From a8dea5f9a31599f24bc7d88c4563fb9271585d2f Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 15 Jul 2026 14:36:26 +0800 Subject: [PATCH] feat(storage): improve storage connectivity error handling and multi-instance support Enhance error messaging for storage connectivity issues by sanitizing raw errors to prevent leaking sensitive information. Introduce a new utility function for consistent error handling across storage backend operations. Additionally, implement logic to support multi-instance storage backends, allowing for more flexible workspace configurations and improved reporting of active storage providers. --- .../src/views/knowledge/KnowledgeBase.vue | 4 + .../application/service/storagebackend.go | 4 +- internal/handler/storagebackend.go | 17 ++- internal/handler/system.go | 101 +++++++++++------- .../handler/system_storage_engine_test.go | 55 ++++++++++ internal/utils/presign.go | 27 ++++- internal/utils/presign_test.go | 7 ++ internal/utils/storage_error.go | 35 ++++++ 8 files changed, 208 insertions(+), 42 deletions(-) create mode 100644 internal/utils/storage_error.go diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index 84ad8216..0bd7ead5 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -154,6 +154,10 @@ onUnmounted(() => { }) const missingStorageEngine = computed(() => { if (!kbInfo.value || isFAQ.value) return false + // storage_backend_id is authoritative; storage_provider_config.provider is a + // compatibility projection for older clients. Either being present means the + // KB has a bound storage instance and uploads should not be blocked. + if (kbInfo.value.storage_backend_id) return false const spc = kbInfo.value.storage_provider_config return !spc || !spc.provider }) diff --git a/internal/application/service/storagebackend.go b/internal/application/service/storagebackend.go index 72c27801..fdad850a 100644 --- a/internal/application/service/storagebackend.go +++ b/internal/application/service/storagebackend.go @@ -35,7 +35,7 @@ func (s *StorageBackendService) Create(ctx context.Context, backend *types.Stora return err } if err := s.Test(ctx, backend); err != nil { - return apperrors.NewBadRequestError("storage connection test failed").WithDetails(err.Error()) + return apperrors.NewBadRequestError("storage connection test failed").WithDetails(secutils.SanitizeStorageConnectivityError(err)) } backend.CreatedAt, backend.UpdatedAt = time.Now(), time.Now() if err := s.repo.Create(ctx, backend); err != nil { @@ -87,7 +87,7 @@ func (s *StorageBackendService) Update(ctx context.Context, incoming *types.Stor return err } if err := s.Test(ctx, incoming); err != nil { - return apperrors.NewBadRequestError("storage connection test failed").WithDetails(err.Error()) + return apperrors.NewBadRequestError("storage connection test failed").WithDetails(secutils.SanitizeStorageConnectivityError(err)) } incoming.UpdatedAt = time.Now() return s.repo.Update(ctx, incoming) diff --git a/internal/handler/storagebackend.go b/internal/handler/storagebackend.go index 9913f6c8..eecd0f22 100644 --- a/internal/handler/storagebackend.go +++ b/internal/handler/storagebackend.go @@ -1,6 +1,7 @@ package handler import ( + "errors" "net/http" apperrors "github.com/Tencent/WeKnora/internal/errors" @@ -10,6 +11,18 @@ import ( "github.com/gin-gonic/gin" ) +// storageTestErrorMessage returns a safe user-facing message for a storage +// connectivity test failure. Validation/AppError messages are already +// user-facing and are passed through, while raw driver/network errors are +// sanitized so they don't leak internal hostnames, IPs, ports or TLS details. +func storageTestErrorMessage(err error) string { + var appErr *apperrors.AppError + if errors.As(err, &appErr) { + return appErr.Message + } + return sanitizeStorageCheckError(err) +} + type StorageBackendHandler struct { repo interfaces.StorageBackendRepository service interfaces.StorageBackendService @@ -121,7 +134,7 @@ func (h *StorageBackendHandler) TestRaw(c *gin.Context) { return } if err := h.service.Test(c.Request.Context(), backend); err != nil { - c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()}) + c.JSON(http.StatusOK, gin.H{"success": false, "error": storageTestErrorMessage(err)}) return } c.JSON(http.StatusOK, gin.H{"success": true}) @@ -138,7 +151,7 @@ func (h *StorageBackendHandler) TestByID(c *gin.Context) { return } if err := h.service.Test(c.Request.Context(), backend); err != nil { - c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()}) + c.JSON(http.StatusOK, gin.H{"success": false, "error": storageTestErrorMessage(err)}) return } c.JSON(http.StatusOK, gin.H{"success": true}) diff --git a/internal/handler/system.go b/internal/handler/system.go index b19b5e9b..b167f276 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -58,6 +58,11 @@ type SystemHandler struct { // runtime task console. It updates business state and tracing before queue // records are removed, unlike a raw Redis deletion. knowledgeSvc runtimeKnowledgeCanceller + // storageBackendRepo lets GetStorageEngineStatus report multi-instance + // storage backends (Settings → Storage) as "available", not just the legacy + // singleton tenant.StorageEngineConfig. Optional — nil in partially-wired + // unit tests, in which case only the legacy config is consulted. + storageBackendRepo interfaces.StorageBackendRepository } // NewSystemHandler creates a new system handler @@ -70,17 +75,19 @@ func NewSystemHandler(cfg *config.Config, auditSvc interfaces.AuditLogService, taskInspector interfaces.TaskInspector, knowledgeSvc interfaces.KnowledgeService, + storageBackendRepo interfaces.StorageBackendRepository, ) *SystemHandler { return &SystemHandler{ - cfg: cfg, - neo4jDriver: neo4jDriver, - documentReader: documentReader, - tenantSvc: tenantSvc, - userSvc: userSvc, - systemSettingSvc: systemSettingSvc, - auditSvc: auditSvc, - taskInspector: taskInspector, - knowledgeSvc: knowledgeSvc, + cfg: cfg, + neo4jDriver: neo4jDriver, + documentReader: documentReader, + tenantSvc: tenantSvc, + userSvc: userSvc, + systemSettingSvc: systemSettingSvc, + auditSvc: auditSvc, + taskInspector: taskInspector, + knowledgeSvc: knowledgeSvc, + storageBackendRepo: storageBackendRepo, } } @@ -502,6 +509,40 @@ func (h *SystemHandler) getMinioConfig(c *gin.Context) (endpoint, accessKeyID, s return } +// activeBackendProviders returns the set of storage providers that have at +// least one active multi-instance backend registered for the caller's +// workspace. Used to keep GetStorageEngineStatus in sync with the new Storage +// settings UI, which writes to storage_backends rather than the legacy +// tenant.StorageEngineConfig singleton. Best-effort and nil-safe: a missing +// repo, missing tenant, or query error yields an empty set so callers fall +// back to the legacy config checks. +func (h *SystemHandler) activeBackendProviders(c *gin.Context) map[string]bool { + result := map[string]bool{} + if h.storageBackendRepo == nil { + return result + } + v, exists := c.Get(types.TenantInfoContextKey.String()) + if !exists { + return result + } + tenant, ok := v.(*types.Tenant) + if !ok || tenant == nil { + return result + } + backends, err := h.storageBackendRepo.List(c.Request.Context(), tenant.ID) + if err != nil { + logger.Warnf(c.Request.Context(), "[storage] list backends for status failed: tenant=%d err=%v", tenant.ID, err) + return result + } + for _, backend := range backends { + if backend == nil || backend.Status != types.StorageBackendStatusActive { + continue + } + result[strings.ToLower(strings.TrimSpace(backend.Provider))] = true + } + return result +} + // isMinioConfigured checks whether MinIO connection info is available (from tenant config or env). func (h *SystemHandler) isMinioConfigured(c *gin.Context) bool { endpoint, accessKeyID, secretAccessKey := h.getMinioConfig(c) @@ -611,14 +652,19 @@ type GetStorageEngineStatusResponse struct { // @Success 200 {object} GetStorageEngineStatusResponse // @Router /system/storage-engine-status [get] func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) { - minioConfigured := h.isMinioConfigured(c) + // Providers that already have an active multi-instance backend registered + // (Settings → Storage). These are authoritative and independent of the + // legacy singleton tenant.StorageEngineConfig, so a workspace that only + // configured storage through the new UI is still reported as available. + activeBackend := h.activeBackendProviders(c) + minioConfigured := h.isMinioConfigured(c) || activeBackend["minio"] minioEnvAvailable := h.isMinioEnvAvailable() - cosConfigured := h.isCOSConfigured(c) - tosConfigured := h.isTOSConfigured(c) - s3Configured := h.isS3Configured(c) - ossConfigured := h.isOSSConfigured(c) - ks3Configured := h.isKS3Configured(c) - obsConfigured := h.isOBSConfigured(c) + cosConfigured := h.isCOSConfigured(c) || activeBackend["cos"] + tosConfigured := h.isTOSConfigured(c) || activeBackend["tos"] + s3Configured := h.isS3Configured(c) || activeBackend["s3"] + ossConfigured := h.isOSSConfigured(c) || activeBackend["oss"] + ks3Configured := h.isKS3Configured(c) || activeBackend["ks3"] + obsConfigured := h.isOBSConfigured(c) || activeBackend["obs"] allowed := getAllowedStorageProviders() allowedProviders := make([]string, 0, len(getSupportedStorageProviders())) for _, provider := range getSupportedStorageProviders() { @@ -652,28 +698,9 @@ var ossFieldPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`) // sanitizeStorageCheckError converts a raw storage connectivity error into a safe // user-facing message that does not leak internal network details (hostnames, IPs, ports). +// The concrete mapping lives in internal/utils so the service layer can share it. func sanitizeStorageCheckError(err error) string { - msg := err.Error() - switch { - case strings.Contains(msg, "Endpoint url cannot have fully qualified paths"): - return "Endpoint 地址格式错误:请去除 http:// 或 https:// 前缀,只填写域名或 IP 地址和端口(例如:minio.example.com:9000)" - case strings.Contains(msg, "no such host"): - return "DNS 解析失败,请检查地址是否正确" - case strings.Contains(msg, "connection refused"): - return "连接被拒绝,请确认服务已启动且端口正确" - case strings.Contains(msg, "no route to host"): - return "无法路由到目标地址,请检查网络配置" - case strings.Contains(msg, "i/o timeout") || strings.Contains(msg, "deadline exceeded") || strings.Contains(msg, "context deadline"): - return "连接超时,请检查网络或服务状态" - case strings.Contains(msg, "403") || strings.Contains(msg, "AccessDenied") || strings.Contains(msg, "access denied"): - return "认证失败,请检查访问凭证是否正确" - case strings.Contains(msg, "certificate") || strings.Contains(msg, "tls") || strings.Contains(msg, "x509"): - return "TLS/SSL 证书错误,请检查 SSL 配置" - case strings.Contains(msg, "404") || strings.Contains(msg, "NoSuchBucket"): - return "Bucket 不存在,请检查名称和 Region" - default: - return "连接失败,请检查配置参数是否正确" - } + return secutils.SanitizeStorageConnectivityError(err) } // storageEndpointHost extracts the hostname from a storage endpoint string. diff --git a/internal/handler/system_storage_engine_test.go b/internal/handler/system_storage_engine_test.go index 4413b32a..645b9793 100644 --- a/internal/handler/system_storage_engine_test.go +++ b/internal/handler/system_storage_engine_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "encoding/json" "net/http" "net/http/httptest" @@ -13,6 +14,25 @@ import ( "github.com/stretchr/testify/require" ) +// fakeStorageBackendRepo is a minimal StorageBackendRepository used to exercise +// GetStorageEngineStatus' multi-instance awareness without a database. +type fakeStorageBackendRepo struct{ backends []*types.StorageBackend } + +func (f *fakeStorageBackendRepo) Create(context.Context, *types.StorageBackend) error { + return nil +} +func (f *fakeStorageBackendRepo) GetByID(context.Context, uint64, string) (*types.StorageBackend, error) { + return nil, nil +} +func (f *fakeStorageBackendRepo) List(context.Context, uint64) ([]*types.StorageBackend, error) { + return f.backends, nil +} +func (f *fakeStorageBackendRepo) Update(context.Context, *types.StorageBackend) error { return nil } +func (f *fakeStorageBackendRepo) Delete(context.Context, uint64, string) error { return nil } +func (f *fakeStorageBackendRepo) FindLegacyAlias(context.Context, uint64, string) (*types.StorageBackend, error) { + return nil, nil +} + func TestGetStorageEngineStatus_IncludesOBS(t *testing.T) { gin.SetMode(gin.TestMode) t.Setenv(storageallowlist.AllowListEnv, "") @@ -90,3 +110,38 @@ func TestGetStorageEngineStatus_OBSConfiguredFromTenant(t *testing.T) { require.NotNil(t, obsStatus) assert.True(t, obsStatus.Available) } + +// A workspace that configured COS only through the new multi-instance Storage +// settings (storage_backends), with an empty legacy StorageEngineConfig, must +// still be reported as available. +func TestGetStorageEngineStatus_AvailableFromActiveBackend(t *testing.T) { + gin.SetMode(gin.TestMode) + t.Setenv(storageallowlist.AllowListEnv, "") + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/storage-engine-status", nil) + c.Set(types.TenantInfoContextKey.String(), &types.Tenant{ID: 42}) + + h := &SystemHandler{storageBackendRepo: &fakeStorageBackendRepo{backends: []*types.StorageBackend{ + {Provider: "cos", Status: types.StorageBackendStatusActive}, + {Provider: "s3", Status: types.StorageBackendStatusDisabled}, + }}} + h.GetStorageEngineStatus(c) + + require.Equal(t, http.StatusOK, w.Code) + + var resp struct { + Data struct { + Engines []StorageEngineStatusItem `json:"engines"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + + status := map[string]bool{} + for _, engine := range resp.Data.Engines { + status[engine.Name] = engine.Available + } + assert.True(t, status["cos"], "active COS backend should be available") + assert.False(t, status["s3"], "disabled S3 backend should not be available") +} diff --git a/internal/utils/presign.go b/internal/utils/presign.go index e7fea5e7..8d31be15 100644 --- a/internal/utils/presign.go +++ b/internal/utils/presign.go @@ -131,12 +131,34 @@ func ValidateKBScopedStoragePath(filePath string, tenantID uint64) error { return nil } +// storageBackendScheme wraps a provider:// path with the concrete instance id: +// storage:///://... It is duplicated here (rather than +// reusing types.ParseStorageBackendPath) because internal/types already imports +// internal/utils, so a reverse import would create a cycle. +const storageBackendScheme = "storage://" + +// unwrapStorageBackendPath strips a leading storage:/// wrapper and +// returns the inner provider:// path. Non-wrapped paths are returned unchanged. +// This keeps tenant/exports parsing anchored on the provider path instead of +// relying on the backend id happening not to look like a tenant segment. +func unwrapStorageBackendPath(filePath string) string { + if !strings.HasPrefix(filePath, storageBackendScheme) { + return filePath + } + rest := strings.TrimPrefix(filePath, storageBackendScheme) + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return filePath + } + return parts[1] +} + // storagePathHasExportsScope reports whether tenantID appears next to an // exports segment in either canonical layout: // - {tenant}/exports/... (local, minio, s3, most cloud backends) // - exports/{tenant}/... (OSS temp-bucket layout) func storagePathHasExportsScope(filePath string, tenantID uint64) bool { - _, rest, ok := strings.Cut(filePath, "://") + _, rest, ok := strings.Cut(unwrapStorageBackendPath(filePath), "://") if !ok { return false } @@ -165,6 +187,9 @@ func storagePathHasExportsScope(filePath string, tenantID uint64) bool { // Callers that have an authoritative resource-owner tenant ID available // should pass it directly to SignFileURL instead of relying on this parser. func ParseTenantIDFromStoragePath(filePath string) uint64 { + // Unwrap storage:/// so the tenant scan is anchored on the inner + // provider path, not the (opaque) backend id. + filePath = unwrapStorageBackendPath(filePath) // Strip scheme: "local://1/abc/img.png" → "1/abc/img.png" _, rest, ok := strings.Cut(filePath, "://") if !ok { diff --git a/internal/utils/presign_test.go b/internal/utils/presign_test.go index 4cb24070..d84406b2 100644 --- a/internal/utils/presign_test.go +++ b/internal/utils/presign_test.go @@ -99,9 +99,14 @@ func TestValidateKBScopedStoragePath(t *testing.T) { assert.NoError(t, ValidateKBScopedStoragePath("minio://bucket/10008/exports/uuid.png", tenantID)) assert.NoError(t, ValidateKBScopedStoragePath("oss://bucket/exports/10008/uuid.png", tenantID)) + // storage:/// wrapped paths must resolve to the same tenant/exports scope. + assert.NoError(t, ValidateKBScopedStoragePath("storage://backend-a/local://10008/exports/img.jpg", tenantID)) + assert.NoError(t, ValidateKBScopedStoragePath("storage://backend-a/cos://bucket/ap-test/10008/exports/a.png", tenantID)) + assert.Error(t, ValidateKBScopedStoragePath("local://10008/knowledge-id/123.pdf", tenantID)) assert.Error(t, ValidateKBScopedStoragePath("local://9999/exports/img.jpg", tenantID)) assert.Error(t, ValidateKBScopedStoragePath("local://10008/other/img.jpg", tenantID)) + assert.Error(t, ValidateKBScopedStoragePath("storage://backend-a/local://9999/exports/img.jpg", tenantID)) } func TestParseTenantIDFromStoragePath(t *testing.T) { @@ -119,6 +124,8 @@ func TestParseTenantIDFromStoragePath(t *testing.T) { {"https://example.com/img.png", 0}, {"invalid", 0}, {"local://exports/file.csv", 0}, + {"storage://backend-a/local://1/abc/img.png", 1}, + {"storage://backend-a/cos://bucket/region/prefix/42/abc/img.png", 42}, } for _, tt := range tests { diff --git a/internal/utils/storage_error.go b/internal/utils/storage_error.go new file mode 100644 index 00000000..02c3e7ba --- /dev/null +++ b/internal/utils/storage_error.go @@ -0,0 +1,35 @@ +package utils + +import "strings" + +// SanitizeStorageConnectivityError converts a raw storage connectivity error +// into a safe, user-facing message. It deliberately avoids echoing the raw +// driver/network error so responses never leak internal hostnames, IPs, ports +// or TLS/certificate details. Callers that need the full error must log it +// server-side instead of returning it to the client. +func SanitizeStorageConnectivityError(err error) string { + if err == nil { + return "" + } + msg := err.Error() + switch { + case strings.Contains(msg, "Endpoint url cannot have fully qualified paths"): + return "Endpoint 地址格式错误:请去除 http:// 或 https:// 前缀,只填写域名或 IP 地址和端口(例如:minio.example.com:9000)" + case strings.Contains(msg, "no such host"): + return "DNS 解析失败,请检查地址是否正确" + case strings.Contains(msg, "connection refused"): + return "连接被拒绝,请确认服务已启动且端口正确" + case strings.Contains(msg, "no route to host"): + return "无法路由到目标地址,请检查网络配置" + case strings.Contains(msg, "i/o timeout") || strings.Contains(msg, "deadline exceeded") || strings.Contains(msg, "context deadline"): + return "连接超时,请检查网络或服务状态" + case strings.Contains(msg, "403") || strings.Contains(msg, "AccessDenied") || strings.Contains(msg, "access denied"): + return "认证失败,请检查访问凭证是否正确" + case strings.Contains(msg, "certificate") || strings.Contains(msg, "tls") || strings.Contains(msg, "x509"): + return "TLS/SSL 证书错误,请检查 SSL 配置" + case strings.Contains(msg, "404") || strings.Contains(msg, "NoSuchBucket"): + return "Bucket 不存在,请检查名称和 Region" + default: + return "连接失败,请检查配置参数是否正确" + } +}