feat(ks3): add KS3 object storage support

Add Kingsoft Cloud KS3 as a supported storage provider, including
  backend service implementation, storage config UI, and provider
  scheme recognition throughout the codebase.
This commit is contained in:
ChenRussell
2026-05-01 19:20:25 +08:00
committed by lyingbug
parent 6f3679716f
commit 42348eb40d
16 changed files with 523 additions and 15 deletions
+1 -1
View File
@@ -7,7 +7,7 @@ import (
const storageAllowListEnv = "STORAGE_ALLOW_LIST"
var supportedStorageProviders = []string{"local", "minio", "cos", "tos", "s3", "oss"}
var supportedStorageProviders = []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3"}
func getSupportedStorageProviders() []string {
providers := make([]string, len(supportedStorageProviders))
+54 -2
View File
@@ -432,6 +432,17 @@ func (h *SystemHandler) isOSSConfigured(c *gin.Context) bool {
return false
}
// isKS3Configured checks whether KS3 connection info is available from tenant config.
func (h *SystemHandler) isKS3Configured(c *gin.Context) bool {
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.KS3 != nil {
ks3Conf := tenant.StorageEngineConfig.KS3
return ks3Conf.Endpoint != "" && ks3Conf.Region != "" && ks3Conf.AccessKey != "" && ks3Conf.SecretKey != "" && ks3Conf.BucketName != ""
}
}
return false
}
// isTOSEnvAvailable checks whether TOS env vars are set.
func (h *SystemHandler) isTOSEnvAvailable() bool {
return os.Getenv("TOS_ENDPOINT") != "" &&
@@ -443,7 +454,7 @@ func (h *SystemHandler) isTOSEnvAvailable() bool {
// StorageEngineStatusItem describes one storage engine's availability and description.
type StorageEngineStatusItem struct {
Name string `json:"name"` // "local", "minio", "cos", "tos"
Name string `json:"name"` // "local", "minio", "cos", "tos"
Allowed bool `json:"allowed"`
Available bool `json:"available"` // whether the engine can be used
Description string `json:"description"` // short description for UI
@@ -470,6 +481,7 @@ func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) {
tosConfigured := h.isTOSConfigured(c)
s3Configured := h.isS3Configured(c)
ossConfigured := h.isOSSConfigured(c)
ks3Configured := h.isKS3Configured(c)
allowed := getAllowedStorageProviders()
allowedProviders := make([]string, 0, len(supportedStorageProviders))
for _, provider := range getSupportedStorageProviders() {
@@ -484,6 +496,7 @@ func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) {
{Name: "tos", Allowed: allowed["tos"], Available: tosConfigured, Description: "火山引擎对象存储服务,适合公有云部署"},
{Name: "s3", Allowed: allowed["s3"], Available: s3Configured, Description: "AWS S3 与兼容对象存储服务,适合公有云与混合云部署"},
{Name: "oss", Allowed: allowed["oss"], Available: ossConfigured, Description: "阿里云对象存储服务,适合公有云部署,支持 S3 兼容协议"},
{Name: "ks3", Allowed: allowed["ks3"], Available: ks3Configured, Description: "金山云对象存储服务,适合公有云部署"},
}
c.JSON(200, gin.H{
"code": 0,
@@ -586,12 +599,13 @@ func isBlockedStorageEndpoint(endpoint string) (bool, string) {
// StorageCheckRequest is the body for POST /system/storage-engine-check.
type StorageCheckRequest struct {
Provider string `json:"provider"` // "minio", "cos", "tos", "s3", "oss"
Provider string `json:"provider"` // "minio", "cos", "tos", "s3", "oss", "ks3"
MinIO *types.MinIOEngineConfig `json:"minio,omitempty"`
COS *types.COSEngineConfig `json:"cos,omitempty"`
TOS *types.TOSEngineConfig `json:"tos,omitempty"`
S3 *types.S3EngineConfig `json:"s3,omitempty"`
OSS *types.OSSEngineConfig `json:"oss,omitempty"`
KS3 *types.KS3EngineConfig `json:"ks3,omitempty"`
}
// StorageCheckResponse is the response for a single-engine connectivity check.
@@ -634,6 +648,8 @@ func (h *SystemHandler) CheckStorageEngine(c *gin.Context) {
h.checkS3(c, ctx, req.S3)
case "oss":
h.checkOSS(c, ctx, req.OSS)
case "ks3":
h.checkKS3(c, ctx, req.KS3)
default:
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: "本地存储无需检测"}})
}
@@ -867,6 +883,42 @@ func (h *SystemHandler) checkOSS(c *gin.Context, ctx context.Context, cfg *types
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}})
}
func (h *SystemHandler) checkKS3(c *gin.Context, ctx context.Context, cfg *types.KS3EngineConfig) {
if cfg == nil {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 KS3 配置"}})
return
}
endpoint, region, accessKey, secretKey := cfg.Endpoint, cfg.Region, cfg.AccessKey, cfg.SecretKey
if endpoint == "" || region == "" || accessKey == "" || secretKey == "" || cfg.BucketName == "" {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Endpoint、Region、Access Key、Secret Key、Bucket 名称不能为空"}})
return
}
if blocked, reason := isBlockedStorageEndpoint(endpoint); blocked {
logger.Warnf(ctx, "Storage check: KS3 endpoint blocked by SSRF protection, endpoint: %s", endpoint)
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}})
return
}
err := file.CheckKS3Connectivity(ctx, endpoint, region, accessKey, secretKey, cfg.BucketName)
if err != nil {
logger.Errorf(ctx, "Storage check: KS3 connectivity failed, bucket: %s, error: %v", cfg.BucketName, err)
errMsg := err.Error()
if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "AccessDenied") {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "认证失败,请检查 Access Key / Secret Key 是否正确"}})
return
}
if strings.Contains(errMsg, "404") || strings.Contains(errMsg, "NoSuchBucket") {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在,请检查名称和 Region", cfg.BucketName)}})
return
}
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: sanitizeStorageCheckError(err)}})
return
}
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}})
}
func (h *SystemHandler) ResolveDocumentReader(ctx context.Context, addr string) interfaces.DocumentReader {
if addr == "" {
return h.documentReader