wizardchen
76a5b329f3
- Updated the StorageCheckResponse to include a new field `bucket_created` to indicate if a bucket was automatically created during the connectivity check. - Modified the MinIO connectivity check logic to attempt auto-creation of the bucket if it does not exist, setting a public-read policy if successful. - Adjusted the frontend to reflect the new response structure, updating the UI to display a 'created' status when a bucket is auto-created. These changes improve the user experience by providing clear feedback on bucket creation during storage checks, enhancing the overall functionality of the storage engine settings.
884 lines
32 KiB
Go
884 lines
32 KiB
Go
package handler
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"github.com/Tencent/WeKnora/internal/application/service/file"
|
|
"github.com/Tencent/WeKnora/internal/config"
|
|
"github.com/Tencent/WeKnora/internal/database"
|
|
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
"github.com/gin-gonic/gin"
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
|
)
|
|
|
|
// SystemHandler handles system-related requests
|
|
type SystemHandler struct {
|
|
cfg *config.Config
|
|
neo4jDriver neo4j.Driver
|
|
documentReader interfaces.DocumentReader
|
|
}
|
|
|
|
// NewSystemHandler creates a new system handler
|
|
func NewSystemHandler(cfg *config.Config, neo4jDriver neo4j.Driver, documentReader interfaces.DocumentReader) *SystemHandler {
|
|
return &SystemHandler{
|
|
cfg: cfg,
|
|
neo4jDriver: neo4jDriver,
|
|
documentReader: documentReader,
|
|
}
|
|
}
|
|
|
|
// GetSystemInfoResponse defines the response structure for system info
|
|
type GetSystemInfoResponse struct {
|
|
Version string `json:"version"`
|
|
Edition string `json:"edition"`
|
|
CommitID string `json:"commit_id,omitempty"`
|
|
BuildTime string `json:"build_time,omitempty"`
|
|
GoVersion string `json:"go_version,omitempty"`
|
|
KeywordIndexEngine string `json:"keyword_index_engine,omitempty"`
|
|
VectorStoreEngine string `json:"vector_store_engine,omitempty"`
|
|
GraphDatabaseEngine string `json:"graph_database_engine,omitempty"`
|
|
MinioEnabled bool `json:"minio_enabled,omitempty"`
|
|
DBVersion string `json:"db_version,omitempty"`
|
|
}
|
|
|
|
// 编译时注入的版本信息
|
|
var (
|
|
Version = "unknown"
|
|
Edition = "standard"
|
|
CommitID = "unknown"
|
|
BuildTime = "unknown"
|
|
GoVersion = "unknown"
|
|
)
|
|
|
|
// GetSystemInfo godoc
|
|
// @Summary 获取系统信息
|
|
// @Description 获取系统版本、构建信息和引擎配置
|
|
// @Tags 系统
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Success 200 {object} GetSystemInfoResponse "系统信息"
|
|
// @Router /system/info [get]
|
|
func (h *SystemHandler) GetSystemInfo(c *gin.Context) {
|
|
ctx := logger.CloneContext(c.Request.Context())
|
|
|
|
// Get keyword index engine from RETRIEVE_DRIVER
|
|
keywordIndexEngine := h.getKeywordIndexEngine()
|
|
|
|
// Get vector store engine from config or RETRIEVE_DRIVER
|
|
vectorStoreEngine := h.getVectorStoreEngine()
|
|
|
|
// Get graph database engine from NEO4J_ENABLE
|
|
graphDatabaseEngine := h.getGraphDatabaseEngine()
|
|
|
|
// Get MinIO enabled status
|
|
minioEnabled := h.isMinioConfigured(c)
|
|
|
|
var dbVersion string
|
|
if ver, dirty, ok := database.CachedMigrationVersion(); ok {
|
|
dbVersion = fmt.Sprintf("%d", ver)
|
|
if dirty {
|
|
dbVersion += " (dirty)"
|
|
}
|
|
}
|
|
|
|
response := GetSystemInfoResponse{
|
|
Version: Version,
|
|
Edition: Edition,
|
|
CommitID: CommitID,
|
|
BuildTime: BuildTime,
|
|
GoVersion: GoVersion,
|
|
KeywordIndexEngine: keywordIndexEngine,
|
|
VectorStoreEngine: vectorStoreEngine,
|
|
GraphDatabaseEngine: graphDatabaseEngine,
|
|
MinioEnabled: minioEnabled,
|
|
DBVersion: dbVersion,
|
|
}
|
|
|
|
logger.Info(ctx, "System info retrieved successfully")
|
|
c.JSON(200, gin.H{
|
|
"code": 0,
|
|
"msg": "success",
|
|
"data": response,
|
|
})
|
|
}
|
|
|
|
func (h *SystemHandler) getDocReaderConnInfo() (addr, transport string) {
|
|
addr = strings.TrimSpace(os.Getenv("DOCREADER_ADDR"))
|
|
transport = strings.TrimSpace(os.Getenv("DOCREADER_TRANSPORT"))
|
|
if transport == "" {
|
|
transport = "grpc"
|
|
}
|
|
transport = strings.ToLower(transport)
|
|
return addr, transport
|
|
}
|
|
|
|
// ListParserEngines returns available document parser engines.
|
|
// Merges Go-native static engines with engines discovered from the remote
|
|
// docreader service, so newly added Python engines are auto-discovered.
|
|
// @Summary 列出可用的文档解析引擎
|
|
// @Tags 系统
|
|
// @Produce json
|
|
// @Success 200 {object} map[string]interface{} "解析引擎列表"
|
|
// @Router /system/parser-engines [get]
|
|
func (h *SystemHandler) ListParserEngines(c *gin.Context) {
|
|
docreaderAddr, docreaderTransport := h.getDocReaderConnInfo()
|
|
connected := h.documentReader != nil && h.documentReader.IsConnected()
|
|
|
|
var overrides map[string]string
|
|
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
|
|
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.ParserEngineConfig != nil {
|
|
overrides = tenant.ParserEngineConfig.ToOverridesMap()
|
|
}
|
|
}
|
|
|
|
remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides)
|
|
engines := docparser.ListAllEngines(connected, overrides, remoteEngines)
|
|
c.JSON(200, gin.H{"code": 0, "msg": "success", "data": engines, "docreader_addr": docreaderAddr, "docreader_transport": docreaderTransport, "connected": connected})
|
|
}
|
|
|
|
// ReconnectDocReader reconnects the document converter to a new (or same) DocReader address.
|
|
// @Summary 重连文档解析服务
|
|
// @Tags 系统
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Param request body object{addr string} true "DocReader 地址"
|
|
// @Success 200
|
|
// @Router /system/docreader/reconnect [post]
|
|
func (h *SystemHandler) ReconnectDocReader(c *gin.Context) {
|
|
var req struct {
|
|
Addr string `json:"addr" binding:"required"`
|
|
}
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
c.JSON(400, gin.H{"code": 1, "msg": "请提供 addr 参数"})
|
|
return
|
|
}
|
|
addr := strings.TrimSpace(req.Addr)
|
|
if addr == "" {
|
|
c.JSON(400, gin.H{"code": 1, "msg": "addr 不能为空"})
|
|
return
|
|
}
|
|
|
|
if h.documentReader == nil {
|
|
c.JSON(500, gin.H{"code": 1, "msg": "document converter not initialized"})
|
|
return
|
|
}
|
|
|
|
if err := h.documentReader.Reconnect(addr); err != nil {
|
|
logger.Errorf(c.Request.Context(), "Failed to reconnect docreader to %s: %v", addr, err)
|
|
c.JSON(200, gin.H{"code": 1, "msg": fmt.Sprintf("连接失败: %v", err)})
|
|
return
|
|
}
|
|
|
|
var overrides map[string]string
|
|
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
|
|
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.ParserEngineConfig != nil {
|
|
overrides = tenant.ParserEngineConfig.ToOverridesMap()
|
|
}
|
|
}
|
|
remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides)
|
|
engines := docparser.ListAllEngines(true, overrides, remoteEngines)
|
|
|
|
_, docreaderTransport := h.getDocReaderConnInfo()
|
|
c.JSON(200, gin.H{"code": 0, "msg": "连接成功", "data": engines, "docreader_addr": addr, "docreader_transport": docreaderTransport, "connected": true})
|
|
}
|
|
|
|
// CheckParserEngines runs availability check with the given config overrides (e.g. current form values).
|
|
// Used to test engine availability without saving; body shape matches ParserEngineConfig.
|
|
// @Summary 使用当前参数检测解析引擎可用性
|
|
// @Tags 系统
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Param body body object true "解析引擎配置（与保存接口同结构）"
|
|
// @Success 200
|
|
// @Router /system/parser-engines/check [post]
|
|
func (h *SystemHandler) CheckParserEngines(c *gin.Context) {
|
|
docreaderAddr, docreaderTransport := h.getDocReaderConnInfo()
|
|
connected := h.documentReader != nil && h.documentReader.IsConnected()
|
|
|
|
var body types.ParserEngineConfig
|
|
if err := c.ShouldBindJSON(&body); err != nil {
|
|
c.JSON(400, gin.H{"code": 1, "msg": "请求体格式错误"})
|
|
return
|
|
}
|
|
overrides := body.ToOverridesMap()
|
|
remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides)
|
|
engines := docparser.ListAllEngines(connected, overrides, remoteEngines)
|
|
c.JSON(200, gin.H{"code": 0, "msg": "success", "data": engines, "docreader_addr": docreaderAddr, "docreader_transport": docreaderTransport, "connected": connected})
|
|
}
|
|
|
|
// fetchRemoteEngines queries the remote docreader for its engine list.
|
|
// Returns nil on any error (e.g. not connected), letting the caller
|
|
// fall back to Go's static registry only.
|
|
func (h *SystemHandler) fetchRemoteEngines(ctx context.Context, overrides map[string]string) []types.ParserEngineInfo {
|
|
if h.documentReader == nil || !h.documentReader.IsConnected() {
|
|
return nil
|
|
}
|
|
engines, err := h.documentReader.ListEngines(ctx, overrides)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "Failed to fetch remote engines from docreader: %v", err)
|
|
return nil
|
|
}
|
|
return engines
|
|
}
|
|
|
|
// getKeywordIndexEngine returns the keyword index engine name
|
|
func (h *SystemHandler) getKeywordIndexEngine() string {
|
|
retrieveDriver := os.Getenv("RETRIEVE_DRIVER")
|
|
if retrieveDriver == "" {
|
|
return "未配置"
|
|
}
|
|
|
|
drivers := strings.Split(retrieveDriver, ",")
|
|
// Filter out engines that support keyword retrieval
|
|
keywordEngines := []string{}
|
|
for _, driver := range drivers {
|
|
driver = strings.TrimSpace(driver)
|
|
if h.supportsRetrieverType(driver, types.KeywordsRetrieverType) {
|
|
keywordEngines = append(keywordEngines, driver)
|
|
}
|
|
}
|
|
|
|
if len(keywordEngines) == 0 {
|
|
return "未配置"
|
|
}
|
|
return strings.Join(keywordEngines, ", ")
|
|
}
|
|
|
|
// getVectorStoreEngine returns the vector store engine name
|
|
func (h *SystemHandler) getVectorStoreEngine() string {
|
|
// First check config.yaml
|
|
if h.cfg != nil && h.cfg.VectorDatabase != nil && h.cfg.VectorDatabase.Driver != "" {
|
|
return h.cfg.VectorDatabase.Driver
|
|
}
|
|
|
|
// Fallback to RETRIEVE_DRIVER for vector support
|
|
retrieveDriver := os.Getenv("RETRIEVE_DRIVER")
|
|
if retrieveDriver == "" {
|
|
return "未配置"
|
|
}
|
|
|
|
drivers := strings.Split(retrieveDriver, ",")
|
|
// Filter out engines that support vector retrieval
|
|
vectorEngines := []string{}
|
|
for _, driver := range drivers {
|
|
driver = strings.TrimSpace(driver)
|
|
if h.supportsRetrieverType(driver, types.VectorRetrieverType) {
|
|
vectorEngines = append(vectorEngines, driver)
|
|
}
|
|
}
|
|
|
|
if len(vectorEngines) == 0 {
|
|
return "未配置"
|
|
}
|
|
return strings.Join(vectorEngines, ", ")
|
|
}
|
|
|
|
// getGraphDatabaseEngine returns the graph database engine name
|
|
func (h *SystemHandler) getGraphDatabaseEngine() string {
|
|
if h.neo4jDriver == nil {
|
|
return "未启用"
|
|
}
|
|
return "Neo4j"
|
|
}
|
|
|
|
// supportsRetrieverType checks if a driver supports a specific retriever type
|
|
// by looking up the retrieverEngineMapping from types package
|
|
func (h *SystemHandler) supportsRetrieverType(driver string, retrieverType types.RetrieverType) bool {
|
|
// Get the mapping of all supported drivers and their capabilities
|
|
mapping := types.GetRetrieverEngineMapping()
|
|
|
|
// Check if the driver exists in the mapping
|
|
engines, exists := mapping[driver]
|
|
if !exists {
|
|
return false
|
|
}
|
|
|
|
// Check if any of the engine configurations support the requested retriever type
|
|
for _, engine := range engines {
|
|
if engine.RetrieverType == retrieverType {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// getMinioConfig resolves MinIO connection parameters from tenant config (if mode=remote) or env vars (mode=docker/default).
|
|
func (h *SystemHandler) getMinioConfig(c *gin.Context) (endpoint, accessKeyID, secretAccessKey string) {
|
|
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
|
|
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
|
|
m := tenant.StorageEngineConfig.MinIO
|
|
if m.Mode == "remote" {
|
|
return m.Endpoint, m.AccessKeyID, m.SecretAccessKey
|
|
}
|
|
}
|
|
}
|
|
endpoint = os.Getenv("MINIO_ENDPOINT")
|
|
accessKeyID = os.Getenv("MINIO_ACCESS_KEY_ID")
|
|
secretAccessKey = os.Getenv("MINIO_SECRET_ACCESS_KEY")
|
|
return
|
|
}
|
|
|
|
// isMinioConfigured checks whether MinIO connection info is available (from tenant config or env).
|
|
func (h *SystemHandler) isMinioConfigured(c *gin.Context) bool {
|
|
endpoint, accessKeyID, secretAccessKey := h.getMinioConfig(c)
|
|
return endpoint != "" && accessKeyID != "" && secretAccessKey != ""
|
|
}
|
|
|
|
// isMinioEnvAvailable checks whether MinIO env vars (MINIO_ENDPOINT etc.) are set.
|
|
func (h *SystemHandler) isMinioEnvAvailable() bool {
|
|
return os.Getenv("MINIO_ENDPOINT") != "" &&
|
|
os.Getenv("MINIO_ACCESS_KEY_ID") != "" &&
|
|
os.Getenv("MINIO_SECRET_ACCESS_KEY") != ""
|
|
}
|
|
|
|
// isCOSConfigured checks whether COS connection info is available from tenant config.
|
|
func (h *SystemHandler) isCOSConfigured(c *gin.Context) bool {
|
|
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
|
|
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.COS != nil {
|
|
cosConf := tenant.StorageEngineConfig.COS
|
|
return cosConf.SecretID != "" && cosConf.SecretKey != "" && cosConf.Region != "" && cosConf.BucketName != ""
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// isTOSConfigured checks whether TOS connection info is available from tenant config or env.
|
|
func (h *SystemHandler) isTOSConfigured(c *gin.Context) bool {
|
|
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
|
|
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.TOS != nil {
|
|
tosConf := tenant.StorageEngineConfig.TOS
|
|
return tosConf.Endpoint != "" && tosConf.Region != "" && tosConf.AccessKey != "" && tosConf.SecretKey != "" && tosConf.BucketName != ""
|
|
}
|
|
}
|
|
return h.isTOSEnvAvailable()
|
|
}
|
|
|
|
// isTOSEnvAvailable checks whether TOS env vars are set.
|
|
func (h *SystemHandler) isTOSEnvAvailable() bool {
|
|
return os.Getenv("TOS_ENDPOINT") != "" &&
|
|
os.Getenv("TOS_REGION") != "" &&
|
|
os.Getenv("TOS_ACCESS_KEY") != "" &&
|
|
os.Getenv("TOS_SECRET_KEY") != "" &&
|
|
os.Getenv("TOS_BUCKET_NAME") != ""
|
|
}
|
|
|
|
// MinioBucketInfo represents bucket information with access policy
|
|
type MinioBucketInfo struct {
|
|
Name string `json:"name"`
|
|
Policy string `json:"policy"` // "public", "private", "custom"
|
|
CreatedAt string `json:"created_at,omitempty"`
|
|
}
|
|
|
|
// ListMinioBucketsResponse defines the response structure for listing buckets
|
|
type ListMinioBucketsResponse struct {
|
|
Buckets []MinioBucketInfo `json:"buckets"`
|
|
}
|
|
|
|
// StorageEngineStatusItem describes one storage engine's availability and description.
|
|
type StorageEngineStatusItem struct {
|
|
Name string `json:"name"` // "local", "minio", "cos", "tos"
|
|
Available bool `json:"available"` // whether the engine can be used
|
|
Description string `json:"description"` // short description for UI
|
|
}
|
|
|
|
// GetStorageEngineStatusResponse is the response for GET /system/storage-engine-status.
|
|
type GetStorageEngineStatusResponse struct {
|
|
Engines []StorageEngineStatusItem `json:"engines"`
|
|
MinioEnvAvailable bool `json:"minio_env_available"`
|
|
}
|
|
|
|
// GetStorageEngineStatus godoc
|
|
// @Summary 获取存储引擎状态
|
|
// @Description 返回 Local、MinIO、COS 各存储引擎的可用状态及说明，供全局设置与知识库选择使用
|
|
// @Tags 系统
|
|
// @Produce json
|
|
// @Success 200 {object} GetStorageEngineStatusResponse
|
|
// @Router /system/storage-engine-status [get]
|
|
func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) {
|
|
minioConfigured := h.isMinioConfigured(c)
|
|
minioEnvAvailable := h.isMinioEnvAvailable()
|
|
cosConfigured := h.isCOSConfigured(c)
|
|
tosConfigured := h.isTOSConfigured(c)
|
|
engines := []StorageEngineStatusItem{
|
|
{Name: "local", Available: true, Description: "本地文件系统存储，仅适合单机部署"},
|
|
{Name: "minio", Available: minioConfigured || minioEnvAvailable, Description: "S3 兼容的自托管对象存储，适合内网和私有云部署"},
|
|
{Name: "cos", Available: cosConfigured, Description: "腾讯云对象存储服务，适合公有云部署，支持 CDN 加速"},
|
|
{Name: "tos", Available: tosConfigured, Description: "火山引擎对象存储服务，适合公有云部署"},
|
|
}
|
|
c.JSON(200, gin.H{
|
|
"code": 0,
|
|
"msg": "success",
|
|
"data": GetStorageEngineStatusResponse{Engines: engines, MinioEnvAvailable: minioEnvAvailable},
|
|
})
|
|
}
|
|
|
|
// ListMinioBuckets godoc
|
|
// @Summary 列出 MinIO 存储桶
|
|
// @Description 获取所有 MinIO 存储桶及其访问权限
|
|
// @Tags 系统
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Success 200 {object} ListMinioBucketsResponse "存储桶列表"
|
|
// @Failure 400 {object} map[string]interface{} "MinIO 未启用"
|
|
// @Failure 500 {object} map[string]interface{} "服务器错误"
|
|
// @Router /system/minio/buckets [get]
|
|
func (h *SystemHandler) ListMinioBuckets(c *gin.Context) {
|
|
ctx := logger.CloneContext(c.Request.Context())
|
|
|
|
endpoint, accessKeyID, secretAccessKey := h.getMinioConfig(c)
|
|
if endpoint == "" || accessKeyID == "" || secretAccessKey == "" {
|
|
logger.Warn(ctx, "MinIO is not configured")
|
|
c.JSON(400, gin.H{
|
|
"code": 400,
|
|
"msg": "MinIO is not configured",
|
|
"success": false,
|
|
})
|
|
return
|
|
}
|
|
|
|
useSSL := os.Getenv("MINIO_USE_SSL") == "true"
|
|
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
|
|
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
|
|
useSSL = tenant.StorageEngineConfig.MinIO.UseSSL
|
|
}
|
|
}
|
|
|
|
// Create MinIO client
|
|
minioClient, err := minio.New(endpoint, &minio.Options{
|
|
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
|
Secure: useSSL,
|
|
})
|
|
if err != nil {
|
|
logger.Error(ctx, "Failed to create MinIO client", "error", err)
|
|
c.JSON(500, gin.H{
|
|
"code": 500,
|
|
"msg": "Failed to connect to MinIO",
|
|
"success": false,
|
|
})
|
|
return
|
|
}
|
|
|
|
// List all buckets
|
|
buckets, err := minioClient.ListBuckets(context.Background())
|
|
if err != nil {
|
|
logger.Error(ctx, "Failed to list MinIO buckets", "error", err)
|
|
c.JSON(500, gin.H{
|
|
"code": 500,
|
|
"msg": "Failed to list buckets",
|
|
"success": false,
|
|
})
|
|
return
|
|
}
|
|
|
|
// Get policy for each bucket
|
|
bucketInfos := make([]MinioBucketInfo, 0, len(buckets))
|
|
for _, bucket := range buckets {
|
|
policy := "private" // default: no policy means private
|
|
|
|
// Try to get bucket policy
|
|
policyStr, err := minioClient.GetBucketPolicy(context.Background(), bucket.Name)
|
|
if err == nil && policyStr != "" {
|
|
policy = parseBucketPolicy(policyStr)
|
|
}
|
|
// If err != nil or policyStr is empty, bucket has no policy (private)
|
|
|
|
bucketInfos = append(bucketInfos, MinioBucketInfo{
|
|
Name: bucket.Name,
|
|
Policy: policy,
|
|
CreatedAt: bucket.CreationDate.Format("2006-01-02 15:04:05"),
|
|
})
|
|
}
|
|
|
|
logger.Info(ctx, "Listed MinIO buckets successfully", "count", len(bucketInfos))
|
|
c.JSON(200, gin.H{
|
|
"code": 0,
|
|
"msg": "success",
|
|
"success": true,
|
|
"data": ListMinioBucketsResponse{Buckets: bucketInfos},
|
|
})
|
|
}
|
|
|
|
// BucketPolicy represents the S3 bucket policy structure
|
|
type BucketPolicy struct {
|
|
Version string `json:"Version"`
|
|
Statement []PolicyStatement `json:"Statement"`
|
|
}
|
|
|
|
// PolicyStatement represents a single statement in the bucket policy
|
|
type PolicyStatement struct {
|
|
Effect string `json:"Effect"`
|
|
Principal interface{} `json:"Principal"` // Can be "*" or {"AWS": [...]}
|
|
Action interface{} `json:"Action"` // Can be string or []string
|
|
Resource interface{} `json:"Resource"` // Can be string or []string
|
|
}
|
|
|
|
// parseBucketPolicy parses the policy JSON and determines the access type
|
|
func parseBucketPolicy(policyStr string) string {
|
|
var policy BucketPolicy
|
|
if err := json.Unmarshal([]byte(policyStr), &policy); err != nil {
|
|
// If we can't parse the policy, treat it as custom
|
|
return "custom"
|
|
}
|
|
|
|
// Check if any statement grants public read access
|
|
hasPublicRead := false
|
|
for _, stmt := range policy.Statement {
|
|
if stmt.Effect != "Allow" {
|
|
continue
|
|
}
|
|
|
|
// Check if Principal is "*" (public)
|
|
if !isPrincipalPublic(stmt.Principal) {
|
|
continue
|
|
}
|
|
|
|
// Check if Action includes s3:GetObject
|
|
if !hasGetObjectAction(stmt.Action) {
|
|
continue
|
|
}
|
|
|
|
hasPublicRead = true
|
|
break
|
|
}
|
|
|
|
if hasPublicRead {
|
|
return "public"
|
|
}
|
|
|
|
// Has policy but not public read
|
|
return "custom"
|
|
}
|
|
|
|
// isPrincipalPublic checks if the principal allows public access
|
|
func isPrincipalPublic(principal interface{}) bool {
|
|
switch p := principal.(type) {
|
|
case string:
|
|
return p == "*"
|
|
case map[string]interface{}:
|
|
// Check for {"AWS": "*"} or {"AWS": ["*"]}
|
|
if aws, ok := p["AWS"]; ok {
|
|
switch a := aws.(type) {
|
|
case string:
|
|
return a == "*"
|
|
case []interface{}:
|
|
for _, v := range a {
|
|
if s, ok := v.(string); ok && s == "*" {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// hasGetObjectAction checks if the action includes s3:GetObject
|
|
func hasGetObjectAction(action interface{}) bool {
|
|
checkAction := func(a string) bool {
|
|
a = strings.ToLower(a)
|
|
return a == "s3:getobject" || a == "s3:*" || a == "*"
|
|
}
|
|
|
|
switch act := action.(type) {
|
|
case string:
|
|
return checkAction(act)
|
|
case []interface{}:
|
|
for _, v := range act {
|
|
if s, ok := v.(string); ok && checkAction(s) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// --- Storage engine helpers ---
|
|
|
|
// cosFieldPattern validates COS region and bucket name format to prevent URL injection.
|
|
var cosFieldPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
|
|
|
|
// sanitizeStorageCheckError converts a raw storage connectivity error into a safe
|
|
// user-facing message that does not leak internal network details (hostnames, IPs, ports).
|
|
func sanitizeStorageCheckError(err error) string {
|
|
msg := err.Error()
|
|
switch {
|
|
case strings.Contains(msg, "no such host"):
|
|
return "DNS 解析失败，请检查地址是否正确"
|
|
case strings.Contains(msg, "connection refused"):
|
|
return "连接被拒绝，请确认服务已启动且端口正确"
|
|
case strings.Contains(msg, "no route to host"):
|
|
return "无法路由到目标地址，请检查网络配置"
|
|
case strings.Contains(msg, "i/o timeout") || strings.Contains(msg, "deadline exceeded") || strings.Contains(msg, "context deadline"):
|
|
return "连接超时，请检查网络或服务状态"
|
|
case strings.Contains(msg, "403") || strings.Contains(msg, "AccessDenied") || strings.Contains(msg, "access denied"):
|
|
return "认证失败，请检查访问凭证是否正确"
|
|
case strings.Contains(msg, "certificate") || strings.Contains(msg, "tls") || strings.Contains(msg, "x509"):
|
|
return "TLS/SSL 证书错误，请检查 SSL 配置"
|
|
case strings.Contains(msg, "404") || strings.Contains(msg, "NoSuchBucket"):
|
|
return "Bucket 不存在，请检查名称和 Region"
|
|
default:
|
|
return "连接失败，请检查配置参数是否正确"
|
|
}
|
|
}
|
|
|
|
// isBlockedStorageEndpoint checks whether a storage endpoint resolves to a dangerous
|
|
// address (cloud metadata, loopback, link-local). Unlike the stricter IsSSRFSafeURL,
|
|
// this allows private IPs since MinIO is commonly deployed on internal networks.
|
|
func isBlockedStorageEndpoint(endpoint string) (bool, string) {
|
|
host, _, err := net.SplitHostPort(endpoint)
|
|
if err != nil {
|
|
host = endpoint
|
|
}
|
|
hostLower := strings.ToLower(host)
|
|
|
|
blockedHosts := []string{
|
|
"metadata.google.internal",
|
|
"metadata.tencentyun.com",
|
|
"metadata.aws.internal",
|
|
}
|
|
for _, bh := range blockedHosts {
|
|
if hostLower == bh {
|
|
return true, "该地址不允许访问"
|
|
}
|
|
}
|
|
|
|
checkIP := func(ip net.IP) (bool, string) {
|
|
if ip.IsLoopback() {
|
|
return true, "不允许访问本地回环地址"
|
|
}
|
|
if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
|
|
return true, "不允许访问链路本地地址"
|
|
}
|
|
if ip.IsUnspecified() {
|
|
return true, "无效的地址"
|
|
}
|
|
return false, ""
|
|
}
|
|
|
|
if ip := net.ParseIP(host); ip != nil {
|
|
return checkIP(ip)
|
|
}
|
|
|
|
ips, err := net.LookupIP(host)
|
|
if err != nil {
|
|
return false, ""
|
|
}
|
|
for _, ip := range ips {
|
|
if blocked, reason := checkIP(ip); blocked {
|
|
return blocked, reason
|
|
}
|
|
}
|
|
return false, ""
|
|
}
|
|
|
|
// --- Storage engine connectivity check ---
|
|
|
|
// StorageCheckRequest is the body for POST /system/storage-engine-check.
|
|
type StorageCheckRequest struct {
|
|
Provider string `json:"provider"` // "minio", "cos", or "tos"
|
|
MinIO *types.MinIOEngineConfig `json:"minio,omitempty"`
|
|
COS *types.COSEngineConfig `json:"cos,omitempty"`
|
|
TOS *types.TOSEngineConfig `json:"tos,omitempty"`
|
|
}
|
|
|
|
// StorageCheckResponse is the response for a single-engine connectivity check.
|
|
type StorageCheckResponse struct {
|
|
OK bool `json:"ok"`
|
|
Message string `json:"message"`
|
|
BucketCreated bool `json:"bucket_created,omitempty"`
|
|
}
|
|
|
|
// CheckStorageEngine tests connectivity for a single storage engine using the provided config.
|
|
// @Summary 测试存储引擎连通性
|
|
// @Description 使用当前填写的参数测试 MinIO/COS 连通性，不保存配置
|
|
// @Tags 系统
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Param body body StorageCheckRequest true "存储引擎配置"
|
|
// @Success 200 {object} StorageCheckResponse
|
|
// @Router /system/storage-engine-check [post]
|
|
func (h *SystemHandler) CheckStorageEngine(c *gin.Context) {
|
|
ctx := logger.CloneContext(c.Request.Context())
|
|
|
|
var req StorageCheckRequest
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
c.JSON(400, gin.H{"code": 1, "msg": "请求体格式错误"})
|
|
return
|
|
}
|
|
|
|
switch req.Provider {
|
|
case "minio":
|
|
h.checkMinio(c, ctx, req.MinIO)
|
|
case "cos":
|
|
h.checkCOS(c, ctx, req.COS)
|
|
case "tos":
|
|
h.checkTOS(c, ctx, req.TOS)
|
|
default:
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: "本地存储无需检测"}})
|
|
}
|
|
}
|
|
|
|
func (h *SystemHandler) checkMinio(c *gin.Context, ctx context.Context, cfg *types.MinIOEngineConfig) {
|
|
if cfg == nil {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 MinIO 配置"}})
|
|
return
|
|
}
|
|
|
|
endpoint, accessKeyID, secretAccessKey := cfg.Endpoint, cfg.AccessKeyID, cfg.SecretAccessKey
|
|
if cfg.Mode != "remote" {
|
|
endpoint = os.Getenv("MINIO_ENDPOINT")
|
|
accessKeyID = os.Getenv("MINIO_ACCESS_KEY_ID")
|
|
secretAccessKey = os.Getenv("MINIO_SECRET_ACCESS_KEY")
|
|
}
|
|
if endpoint == "" || accessKeyID == "" || secretAccessKey == "" {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Endpoint、Access Key、Secret Key 不能为空"}})
|
|
return
|
|
}
|
|
|
|
if cfg.Mode == "remote" {
|
|
if blocked, reason := isBlockedStorageEndpoint(endpoint); blocked {
|
|
logger.Warnf(ctx, "Storage check: MinIO endpoint blocked by SSRF protection", "endpoint", endpoint)
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}})
|
|
return
|
|
}
|
|
}
|
|
|
|
err := file.CheckMinioConnectivity(ctx, endpoint, accessKeyID, secretAccessKey, cfg.BucketName, cfg.UseSSL)
|
|
if err != nil {
|
|
errMsg := err.Error()
|
|
// If bucket does not exist, auto-create it with public-read policy
|
|
if strings.Contains(errMsg, "does not exist") && cfg.BucketName != "" {
|
|
logger.Info(ctx, "Storage check: bucket does not exist, attempting auto-creation", "bucket", cfg.BucketName)
|
|
minioClient, clientErr := minio.New(endpoint, &minio.Options{
|
|
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
|
Secure: cfg.UseSSL,
|
|
})
|
|
if clientErr != nil {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("创建 MinIO 客户端失败: %s", sanitizeStorageCheckError(clientErr))}})
|
|
return
|
|
}
|
|
if mkErr := minioClient.MakeBucket(ctx, cfg.BucketName, minio.MakeBucketOptions{}); mkErr != nil {
|
|
logger.Error(ctx, "Storage check: failed to create bucket", "bucket", cfg.BucketName, "error", mkErr)
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("自动创建 Bucket「%s」失败: %s", cfg.BucketName, sanitizeStorageCheckError(mkErr))}})
|
|
return
|
|
}
|
|
// Set public-read policy
|
|
publicReadPolicy := fmt.Sprintf(`{
|
|
"Version": "2012-10-17",
|
|
"Statement": [
|
|
{
|
|
"Effect": "Allow",
|
|
"Principal": {"AWS": ["*"]},
|
|
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
|
"Resource": ["arn:aws:s3:::%s"]
|
|
},
|
|
{
|
|
"Effect": "Allow",
|
|
"Principal": {"AWS": ["*"]},
|
|
"Action": ["s3:GetObject"],
|
|
"Resource": ["arn:aws:s3:::%s/*"]
|
|
}
|
|
]
|
|
}`, cfg.BucketName, cfg.BucketName)
|
|
if policyErr := minioClient.SetBucketPolicy(ctx, cfg.BucketName, publicReadPolicy); policyErr != nil {
|
|
logger.Error(ctx, "Storage check: bucket created but failed to set public-read policy", "bucket", cfg.BucketName, "error", policyErr)
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, BucketCreated: true, Message: fmt.Sprintf("Bucket「%s」已自动创建，但设置公有读策略失败，请手动配置权限", cfg.BucketName)}})
|
|
return
|
|
}
|
|
logger.Info(ctx, "Storage check: bucket created with public-read policy", "bucket", cfg.BucketName)
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, BucketCreated: true, Message: fmt.Sprintf("Bucket「%s」不存在，已自动创建并设置公有读权限", cfg.BucketName)}})
|
|
return
|
|
}
|
|
logger.Error(ctx, "Storage check: MinIO connectivity failed", "error", err)
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: sanitizeStorageCheckError(err)}})
|
|
return
|
|
}
|
|
|
|
msg := "连接成功"
|
|
if cfg.BucketName != "" {
|
|
msg = fmt.Sprintf("连接成功，Bucket「%s」已确认存在", cfg.BucketName)
|
|
}
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: msg}})
|
|
}
|
|
|
|
func (h *SystemHandler) checkCOS(c *gin.Context, ctx context.Context, cfg *types.COSEngineConfig) {
|
|
if cfg == nil {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 COS 配置"}})
|
|
return
|
|
}
|
|
if cfg.SecretID == "" || cfg.SecretKey == "" || cfg.Region == "" || cfg.BucketName == "" {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Secret ID、Secret Key、Region、Bucket 名称不能为空"}})
|
|
return
|
|
}
|
|
if !cosFieldPattern.MatchString(cfg.Region) {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Region 格式不正确，仅允许字母、数字、点、连字符"}})
|
|
return
|
|
}
|
|
if !cosFieldPattern.MatchString(cfg.BucketName) {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Bucket 名称格式不正确，仅允许字母、数字、点、连字符"}})
|
|
return
|
|
}
|
|
|
|
err := file.CheckCosConnectivity(ctx, cfg.BucketName, cfg.Region, cfg.SecretID, cfg.SecretKey)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Storage check: COS connectivity failed, bucket: %s, error: %v", cfg.BucketName, err)
|
|
errMsg := err.Error()
|
|
if strings.Contains(errMsg, "403") {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "认证失败，请检查 Secret ID / Secret Key 是否正确"}})
|
|
return
|
|
}
|
|
if strings.Contains(errMsg, "404") || strings.Contains(errMsg, "NoSuchBucket") {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在，请检查名称和 Region", cfg.BucketName)}})
|
|
return
|
|
}
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: sanitizeStorageCheckError(err)}})
|
|
return
|
|
}
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功，Bucket「%s」已确认存在", cfg.BucketName)}})
|
|
}
|
|
|
|
func (h *SystemHandler) checkTOS(c *gin.Context, ctx context.Context, cfg *types.TOSEngineConfig) {
|
|
if cfg == nil {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 TOS 配置"}})
|
|
return
|
|
}
|
|
if cfg.Endpoint == "" || cfg.Region == "" || cfg.AccessKey == "" || cfg.SecretKey == "" || cfg.BucketName == "" {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Endpoint、Region、Access Key、Secret Key、Bucket 名称不能为空"}})
|
|
return
|
|
}
|
|
|
|
if blocked, reason := isBlockedStorageEndpoint(cfg.Endpoint); blocked {
|
|
logger.Warnf(ctx, "Storage check: TOS endpoint blocked by SSRF protection, endpoint: %s", cfg.Endpoint)
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}})
|
|
return
|
|
}
|
|
|
|
err := file.CheckTosConnectivity(ctx, cfg.Endpoint, cfg.Region, cfg.AccessKey, cfg.SecretKey, cfg.BucketName)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Storage check: TOS connectivity failed, bucket: %s, error: %v", cfg.BucketName, err)
|
|
errMsg := err.Error()
|
|
if strings.Contains(errMsg, "403") {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "认证失败，请检查 Access Key / Secret Key 是否正确"}})
|
|
return
|
|
}
|
|
if strings.Contains(errMsg, "404") {
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在，请检查名称和 Region", cfg.BucketName)}})
|
|
return
|
|
}
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: sanitizeStorageCheckError(err)}})
|
|
return
|
|
}
|
|
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功，Bucket「%s」已确认存在", cfg.BucketName)}})
|
|
}
|