From 42348eb40d847a535c24a436019d4d50e353e23d Mon Sep 17 00:00:00 2001 From: ChenRussell <792608290@qq.com> Date: Fri, 1 May 2026 19:20:25 +0800 Subject: [PATCH] feat(ks3): add KS3 object storage support Add Kingsoft Cloud KS3 as a supported storage provider, including backend service implementation, storage config UI, and provider scheme recognition throughout the codebase. --- frontend/src/api/system/index.ts | 13 +- frontend/src/i18n/locales/en-US.ts | 10 + frontend/src/i18n/locales/ko-KR.ts | 10 + frontend/src/i18n/locales/ru-RU.ts | 10 + frontend/src/i18n/locales/zh-CN.ts | 10 + frontend/src/utils/security.ts | 10 +- .../knowledge/settings/KBStorageSettings.vue | 10 +- .../views/settings/StorageEngineSettings.vue | 118 ++++++++ go.mod | 1 + go.sum | 3 + internal/application/service/file/factory.go | 11 + internal/application/service/file/ks3.go | 255 ++++++++++++++++++ internal/handler/storage_allowlist.go | 2 +- internal/handler/system.go | 56 +++- internal/types/knowledgebase.go | 4 +- internal/types/tenant.go | 15 +- 16 files changed, 523 insertions(+), 15 deletions(-) create mode 100644 internal/application/service/file/ks3.go diff --git a/frontend/src/api/system/index.ts b/frontend/src/api/system/index.ts index e0aa5269..4987cfe0 100644 --- a/frontend/src/api/system/index.ts +++ b/frontend/src/api/system/index.ts @@ -169,7 +169,7 @@ export function reconnectDocReader(addr: string): Promise]*?)\ssrc=(["'])(local|minio|cos|tos|s3|oss):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi, + /]*?)\ssrc=(["'])(local|minio|cos|tos|s3|oss|ks3):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi, (_m, before, quote, provider, restPathRaw, after) => { const restPath = decodeProviderURL(restPathRaw); const protectedSrc = `${provider}://${restPath}`; @@ -178,7 +178,7 @@ export function isValidURL(url: string): boolean { } // 允许 provider:// 形式,由前端后续鉴权拉取并替换为 blob URL - if (/^(local|minio|cos|tos|s3|oss):\/\/\S+$/i.test(trimmed)) { + if (/^(local|minio|cos|tos|s3|oss|ks3):\/\/\S+$/i.test(trimmed)) { return true; } @@ -304,7 +304,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef } const images = root.querySelectorAll( - 'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"]', + 'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"], img[src^="ks3://"]', ); if (!images.length) { return; @@ -324,7 +324,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef } img.dataset.authHydrated = '1'; - const isProviderScheme = /^(local|minio|cos|tos|s3|oss):\/\//.test(sourceURL); + const isProviderScheme = /^(local|minio|cos|tos|s3|oss|ks3):\/\//.test(sourceURL); const requestURL = isProviderScheme ? `/files?${new URLSearchParams({ file_path: sourceURL }).toString()}` : sourceURL; diff --git a/frontend/src/views/knowledge/settings/KBStorageSettings.vue b/frontend/src/views/knowledge/settings/KBStorageSettings.vue index 36f5a7bc..be5b13fb 100644 --- a/frontend/src/views/knowledge/settings/KBStorageSettings.vue +++ b/frontend/src/views/knowledge/settings/KBStorageSettings.vue @@ -131,6 +131,14 @@ const engineOptions = computed(() => { available: statusMap.oss, disabled: allowedMap.oss === false || statusMap.oss === false, }, + { + value: 'ks3', + label: t('kbSettings.storage.engineKs3'), + desc: t('kbSettings.storage.engineKs3Desc'), + allowed: allowedMap.ks3 !== false, + available: statusMap.ks3, + disabled: allowedMap.ks3 === false || statusMap.ks3 === false, + }, ] }) @@ -171,7 +179,7 @@ async function load() { allowedProviders.value = statusRes?.data?.allowed_providers ?? [] defaultProvider.value = configRes?.data?.default_provider || 'local' const d = configRes?.data - hasAnyConfig.value = !!(d?.local?.path_prefix || d?.minio?.bucket_name || d?.cos?.bucket_name || d?.tos?.bucket_name || d?.s3?.bucket_name) + hasAnyConfig.value = !!(d?.local?.path_prefix || d?.minio?.bucket_name || d?.cos?.bucket_name || d?.tos?.bucket_name || d?.s3?.bucket_name || d?.oss?.bucket_name || d?.ks3?.bucket_name) if (!localProvider.value || localProvider.value === '') { localProvider.value = defaultProvider.value emit('update:storageProvider', localProvider.value) diff --git a/frontend/src/views/settings/StorageEngineSettings.vue b/frontend/src/views/settings/StorageEngineSettings.vue index 4c7f57a3..434e531a 100644 --- a/frontend/src/views/settings/StorageEngineSettings.vue +++ b/frontend/src/views/settings/StorageEngineSettings.vue @@ -123,6 +123,18 @@

{{ $t('settings.storage.ossDesc') }}

+ +
+
+

{{ $t('settings.storage.ks3Title') }}

+ {{ $t('settings.storage.configurable') }} +
+

{{ $t('settings.storage.ks3Desc') }}

+
@@ -378,6 +390,65 @@ + +
@@ -432,6 +503,14 @@ const defaultConfig = (): StorageEngineConfig => ({ temp_bucket_name: '', temp_region: '', }, + ks3: { + endpoint: '', + region: '', + access_key: '', + secret_key: '', + bucket_name: '', + path_prefix: '', + }, }) const loading = ref(true) @@ -454,6 +533,8 @@ const checkingS3 = ref(false) const s3CheckResult = ref<{ ok: boolean; message: string } | null>(null) const checkingOss = ref(false) const ossCheckResult = ref<{ ok: boolean; message: string } | null>(null) +const checkingKs3 = ref(false) +const ks3CheckResult = ref<{ ok: boolean; message: string } | null>(null) const drawerVisible = ref(false) const currentEngine = ref(null) @@ -465,6 +546,7 @@ const providerOptions = computed(() => [ { value: 'tos', label: t('settings.storage.engineTos'), allowed: isProviderAllowed('tos') }, { value: 's3', label: 'AWS S3', allowed: isProviderAllowed('s3') }, { value: 'oss', label: t('settings.storage.engineOss'), allowed: isProviderAllowed('oss') }, + { value: 'ks3', label: t('settings.storage.engineKs3'), allowed: isProviderAllowed('ks3') }, ]) const hasAllowedProviders = computed(() => (allowedProviders.value?.length ?? 0) > 0) @@ -481,6 +563,8 @@ const currentCheckState = computed(() => { return { loading: checkingS3.value, result: s3CheckResult.value, onCheck: onCheckS3 } case 'oss': return { loading: checkingOss.value, result: ossCheckResult.value, onCheck: onCheckOss } + case 'ks3': + return { loading: checkingKs3.value, result: ks3CheckResult.value, onCheck: onCheckKs3 } default: return { loading: false, result: null, onCheck: () => undefined } } @@ -495,6 +579,7 @@ const drawerTitle = computed(() => { tos: t('settings.storage.tosTitle'), s3: t('settings.storage.s3Title'), oss: t('settings.storage.ossTitle'), + ks3: t('settings.storage.ks3Title'), } return titles[currentEngine.value] || currentEngine.value }) @@ -526,6 +611,7 @@ function openDrawer(engine: string) { tosCheckResult.value = null s3CheckResult.value = null ossCheckResult.value = null + ks3CheckResult.value = null } async function loadConfig() { @@ -590,6 +676,16 @@ async function loadConfig() { temp_region: d.oss.temp_region || '', } : defaultConfig().oss, + ks3: d.ks3 + ? { + endpoint: d.ks3.endpoint || '', + region: d.ks3.region || '', + access_key: d.ks3.access_key || '', + secret_key: d.ks3.secret_key || '', + bucket_name: d.ks3.bucket_name || '', + path_prefix: d.ks3.path_prefix || '', + } + : defaultConfig().ks3, } } } catch { @@ -681,6 +777,14 @@ function buildPayload(): StorageEngineConfig { temp_bucket_name: (config.value.oss?.temp_bucket_name || '').trim(), temp_region: (config.value.oss?.temp_region || '').trim(), }, + ks3: { + endpoint: (config.value.ks3?.endpoint || '').trim(), + region: (config.value.ks3?.region || '').trim(), + access_key: (config.value.ks3?.access_key || '').trim(), + secret_key: (config.value.ks3?.secret_key || '').trim(), + bucket_name: (config.value.ks3?.bucket_name || '').trim(), + path_prefix: (config.value.ks3?.path_prefix || '').trim(), + }, } } @@ -777,6 +881,20 @@ async function onCheckOss() { } } +async function onCheckKs3() { + checkingKs3.value = true + ks3CheckResult.value = null + try { + const payload = buildPayload() + const res = await checkStorageEngine({ provider: 'ks3', ks3: payload.ks3 }) + ks3CheckResult.value = res?.data ?? { ok: false, message: t('settings.storage.unknownError') } + } catch (e: unknown) { + ks3CheckResult.value = { ok: false, message: e instanceof Error ? e.message : t('settings.storage.requestFailed') } + } finally { + checkingKs3.value = false + } +} + onMounted(loadAll) diff --git a/go.mod b/go.mod index 7fc88f9e..3a354f13 100644 --- a/go.mod +++ b/go.mod @@ -204,6 +204,7 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/ks3sdklib/aws-sdk-go v1.11.0 // indirect github.com/labstack/echo/v4 v4.13.3 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/leaanthony/go-ansi-parser v1.6.1 // indirect diff --git a/go.sum b/go.sum index 6fe43268..5315e945 100644 --- a/go.sum +++ b/go.sum @@ -2099,6 +2099,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ks3sdklib/aws-sdk-go v1.11.0 h1:CTYXt7KzMdbm59h0a3DIWZN1jN2Qi5SP5OjCyWX01vE= +github.com/ks3sdklib/aws-sdk-go v1.11.0/go.mod h1:fOYkqRwykBT2tHD/UkZ2qHKxEtKNJd2gJVt2fveKRk0= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= @@ -2718,6 +2720,7 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= diff --git a/internal/application/service/file/factory.go b/internal/application/service/file/factory.go index a726b0f9..1ca115ba 100644 --- a/internal/application/service/file/factory.go +++ b/internal/application/service/file/factory.go @@ -126,6 +126,17 @@ func NewFileServiceFromStorageConfig( } return svc, p, err + case "ks3": + if sec == nil || sec.KS3 == nil || sec.KS3.Endpoint == "" || sec.KS3.Region == "" || sec.KS3.AccessKey == "" || sec.KS3.SecretKey == "" || sec.KS3.BucketName == "" { + return nil, p, fmt.Errorf("incomplete ks3 config") + } + pathPrefix := strings.TrimSpace(sec.KS3.PathPrefix) + if pathPrefix == "" { + pathPrefix = "weknora/" + } + svc, err := NewKS3FileService(sec.KS3.Endpoint, sec.KS3.Region, sec.KS3.AccessKey, sec.KS3.SecretKey, sec.KS3.BucketName, pathPrefix) + return svc, p, err + default: return nil, p, fmt.Errorf("unsupported provider %q", p) } diff --git a/internal/application/service/file/ks3.go b/internal/application/service/file/ks3.go new file mode 100644 index 00000000..cb4ceb84 --- /dev/null +++ b/internal/application/service/file/ks3.go @@ -0,0 +1,255 @@ +package file + +import ( + "bytes" + "context" + "fmt" + "io" + "mime/multipart" + "path/filepath" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" + "github.com/google/uuid" + ks3aws "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/aws/credentials" + ks3s3 "github.com/ks3sdklib/aws-sdk-go/service/s3" +) + +const ks3Scheme = "ks3://" + +// ks3FileService implements FileService for Kingsoft Cloud KS3. +// KS3 uses V2 signing by default and virtual-hosted style addressing, +// so it cannot be handled by the generic S3 provider without workarounds. +type ks3FileService struct { + client *ks3s3.S3 + bucketName string + pathPrefix string +} + +// NewKS3FileService creates a KS3 file service and ensures the bucket exists. +func NewKS3FileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) { + client, err := newKS3Client(endpoint, region, accessKey, secretKey) + if err != nil { + return nil, err + } + + pathPrefix = strings.Trim(pathPrefix, "/") + + svc := &ks3FileService{ + client: client, + bucketName: bucketName, + pathPrefix: pathPrefix, + } + + if err := ensureKS3Bucket(client, bucketName); err != nil { + return nil, err + } + + return svc, nil +} + +func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, error) { + creds := credentials.NewStaticCredentials(accessKey, secretKey, "") + client := ks3s3.New(&ks3aws.Config{ + Credentials: creds, + Region: region, + Endpoint: endpoint, + DisableSSL: false, + S3ForcePathStyle: false, // KS3 uses virtual-hosted style + SignerVersion: "V2", // KS3 recommends V2 signing + MaxRetries: 3, + }) + return client, nil +} + +func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error { + _, err := client.HeadBucket(&ks3s3.HeadBucketInput{ + Bucket: ks3aws.String(bucketName), + }) + if err == nil { + return nil + } + // Bucket doesn't exist, try to create it + _, createErr := client.CreateBucket(&ks3s3.CreateBucketInput{ + Bucket: ks3aws.String(bucketName), + }) + if createErr != nil { + return fmt.Errorf("failed to create KS3 bucket %q: %w", bucketName, createErr) + } + return nil +} + +// CheckKS3Connectivity tests KS3 connectivity using the provided credentials. +func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error { + client, err := newKS3Client(endpoint, region, accessKey, secretKey) + if err != nil { + return err + } + + done := make(chan error, 1) + go func() { + _, err := client.HeadBucket(&ks3s3.HeadBucketInput{ + Bucket: ks3aws.String(bucketName), + }) + done <- err + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-done: + return err + } +} + +func joinKS3Key(parts ...string) string { + filtered := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.Trim(p, "/") + if p != "" { + filtered = append(filtered, p) + } + } + return strings.Join(filtered, "/") +} + +func parseKS3FilePath(filePath string) (bucket, objectKey string, err error) { + if !strings.HasPrefix(filePath, ks3Scheme) { + return "", "", fmt.Errorf("invalid KS3 file path: %s", filePath) + } + rest := strings.TrimPrefix(filePath, ks3Scheme) + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid KS3 file path: %s", filePath) + } + return parts[0], parts[1], nil +} + +func (s *ks3FileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) { + ext := filepath.Ext(file.Filename) + objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), knowledgeID, uuid.New().String()+ext) + + src, err := file.Open() + if err != nil { + return "", fmt.Errorf("failed to open file: %w", err) + } + defer src.Close() + + contentType := file.Header.Get("Content-Type") + if contentType == "" { + contentType = utils.GetContentTypeByExt(ext) + } + + _, err = s.client.PutObject(&ks3s3.PutObjectInput{ + Bucket: ks3aws.String(s.bucketName), + Key: ks3aws.String(objectKey), + Body: src, + ContentType: ks3aws.String(contentType), + }) + if err != nil { + return "", fmt.Errorf("failed to upload file to KS3: %w", err) + } + + return fmt.Sprintf("%s%s/%s", ks3Scheme, s.bucketName, objectKey), nil +} + +func (s *ks3FileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) { + safeName, err := utils.SafeFileName(fileName) + if err != nil { + return "", fmt.Errorf("invalid file name: %w", err) + } + ext := filepath.Ext(safeName) + objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), "exports", uuid.New().String()+ext) + + _, err = s.client.PutObject(&ks3s3.PutObjectInput{ + Bucket: ks3aws.String(s.bucketName), + Key: ks3aws.String(objectKey), + Body: bytes.NewReader(data), + ContentType: ks3aws.String(utils.GetContentTypeByExt(ext)), + }) + if err != nil { + return "", fmt.Errorf("failed to upload bytes to KS3: %w", err) + } + + return fmt.Sprintf("%s%s/%s", ks3Scheme, s.bucketName, objectKey), nil +} + +func (s *ks3FileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { + _, objectKey, err := parseKS3FilePath(filePath) + if err != nil { + return nil, err + } + if err := utils.SafeObjectKey(objectKey); err != nil { + return nil, fmt.Errorf("invalid file path: %w", err) + } + + resp, err := s.client.GetObject(&ks3s3.GetObjectInput{ + Bucket: ks3aws.String(s.bucketName), + Key: ks3aws.String(objectKey), + }) + if err != nil { + return nil, fmt.Errorf("failed to get file from KS3: %w", err) + } + + return resp.Body, nil +} + +func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error { + _, objectKey, err := parseKS3FilePath(filePath) + if err != nil { + return err + } + if err := utils.SafeObjectKey(objectKey); err != nil { + return fmt.Errorf("invalid file path: %w", err) + } + + _, err = s.client.DeleteObject(&ks3s3.DeleteObjectInput{ + Bucket: ks3aws.String(s.bucketName), + Key: ks3aws.String(objectKey), + }) + if err != nil { + return fmt.Errorf("failed to delete file from KS3: %w", err) + } + return nil +} + +func (s *ks3FileService) CheckConnectivity(ctx context.Context) error { + done := make(chan error, 1) + go func() { + _, err := s.client.HeadBucket(&ks3s3.HeadBucketInput{ + Bucket: ks3aws.String(s.bucketName), + }) + done <- err + }() + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-done: + return err + } +} + +func (s *ks3FileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + _, objectKey, err := parseKS3FilePath(filePath) + if err != nil { + return "", err + } + if err := utils.SafeObjectKey(objectKey); err != nil { + return "", fmt.Errorf("invalid file path: %w", err) + } + + url, err := s.client.GeneratePresignedUrl(&ks3s3.GeneratePresignedUrlInput{ + Bucket: ks3aws.String(s.bucketName), + Key: ks3aws.String(objectKey), + HTTPMethod: ks3s3.HTTPMethod("GET"), + Expires: int64((24 * time.Hour).Seconds()), + }) + if err != nil { + return "", fmt.Errorf("failed to generate KS3 presigned URL: %w", err) + } + + return url, nil +} diff --git a/internal/handler/storage_allowlist.go b/internal/handler/storage_allowlist.go index 5bfd64ff..b4d583e5 100644 --- a/internal/handler/storage_allowlist.go +++ b/internal/handler/storage_allowlist.go @@ -7,7 +7,7 @@ import ( const storageAllowListEnv = "STORAGE_ALLOW_LIST" -var supportedStorageProviders = []string{"local", "minio", "cos", "tos", "s3", "oss"} +var supportedStorageProviders = []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3"} func getSupportedStorageProviders() []string { providers := make([]string, len(supportedStorageProviders)) diff --git a/internal/handler/system.go b/internal/handler/system.go index db04afa6..1aec5e11 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -432,6 +432,17 @@ func (h *SystemHandler) isOSSConfigured(c *gin.Context) bool { return false } +// isKS3Configured checks whether KS3 connection info is available from tenant config. +func (h *SystemHandler) isKS3Configured(c *gin.Context) bool { + if v, exists := c.Get(types.TenantInfoContextKey.String()); exists { + if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.KS3 != nil { + ks3Conf := tenant.StorageEngineConfig.KS3 + return ks3Conf.Endpoint != "" && ks3Conf.Region != "" && ks3Conf.AccessKey != "" && ks3Conf.SecretKey != "" && ks3Conf.BucketName != "" + } + } + return false +} + // isTOSEnvAvailable checks whether TOS env vars are set. func (h *SystemHandler) isTOSEnvAvailable() bool { return os.Getenv("TOS_ENDPOINT") != "" && @@ -443,7 +454,7 @@ func (h *SystemHandler) isTOSEnvAvailable() bool { // StorageEngineStatusItem describes one storage engine's availability and description. type StorageEngineStatusItem struct { - Name string `json:"name"` // "local", "minio", "cos", "tos" + Name string `json:"name"` // "local", "minio", "cos", "tos" Allowed bool `json:"allowed"` Available bool `json:"available"` // whether the engine can be used Description string `json:"description"` // short description for UI @@ -470,6 +481,7 @@ func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) { tosConfigured := h.isTOSConfigured(c) s3Configured := h.isS3Configured(c) ossConfigured := h.isOSSConfigured(c) + ks3Configured := h.isKS3Configured(c) allowed := getAllowedStorageProviders() allowedProviders := make([]string, 0, len(supportedStorageProviders)) for _, provider := range getSupportedStorageProviders() { @@ -484,6 +496,7 @@ func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) { {Name: "tos", Allowed: allowed["tos"], Available: tosConfigured, Description: "火山引擎对象存储服务,适合公有云部署"}, {Name: "s3", Allowed: allowed["s3"], Available: s3Configured, Description: "AWS S3 与兼容对象存储服务,适合公有云与混合云部署"}, {Name: "oss", Allowed: allowed["oss"], Available: ossConfigured, Description: "阿里云对象存储服务,适合公有云部署,支持 S3 兼容协议"}, + {Name: "ks3", Allowed: allowed["ks3"], Available: ks3Configured, Description: "金山云对象存储服务,适合公有云部署"}, } c.JSON(200, gin.H{ "code": 0, @@ -586,12 +599,13 @@ func isBlockedStorageEndpoint(endpoint string) (bool, string) { // StorageCheckRequest is the body for POST /system/storage-engine-check. type StorageCheckRequest struct { - Provider string `json:"provider"` // "minio", "cos", "tos", "s3", "oss" + Provider string `json:"provider"` // "minio", "cos", "tos", "s3", "oss", "ks3" MinIO *types.MinIOEngineConfig `json:"minio,omitempty"` COS *types.COSEngineConfig `json:"cos,omitempty"` TOS *types.TOSEngineConfig `json:"tos,omitempty"` S3 *types.S3EngineConfig `json:"s3,omitempty"` OSS *types.OSSEngineConfig `json:"oss,omitempty"` + KS3 *types.KS3EngineConfig `json:"ks3,omitempty"` } // StorageCheckResponse is the response for a single-engine connectivity check. @@ -634,6 +648,8 @@ func (h *SystemHandler) CheckStorageEngine(c *gin.Context) { h.checkS3(c, ctx, req.S3) case "oss": h.checkOSS(c, ctx, req.OSS) + case "ks3": + h.checkKS3(c, ctx, req.KS3) default: c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: "本地存储无需检测"}}) } @@ -867,6 +883,42 @@ func (h *SystemHandler) checkOSS(c *gin.Context, ctx context.Context, cfg *types c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}}) } +func (h *SystemHandler) checkKS3(c *gin.Context, ctx context.Context, cfg *types.KS3EngineConfig) { + if cfg == nil { + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 KS3 配置"}}) + return + } + + endpoint, region, accessKey, secretKey := cfg.Endpoint, cfg.Region, cfg.AccessKey, cfg.SecretKey + if endpoint == "" || region == "" || accessKey == "" || secretKey == "" || cfg.BucketName == "" { + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Endpoint、Region、Access Key、Secret Key、Bucket 名称不能为空"}}) + return + } + + if blocked, reason := isBlockedStorageEndpoint(endpoint); blocked { + logger.Warnf(ctx, "Storage check: KS3 endpoint blocked by SSRF protection, endpoint: %s", endpoint) + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}}) + return + } + + err := file.CheckKS3Connectivity(ctx, endpoint, region, accessKey, secretKey, cfg.BucketName) + if err != nil { + logger.Errorf(ctx, "Storage check: KS3 connectivity failed, bucket: %s, error: %v", cfg.BucketName, err) + errMsg := err.Error() + if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "AccessDenied") { + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "认证失败,请检查 Access Key / Secret Key 是否正确"}}) + return + } + if strings.Contains(errMsg, "404") || strings.Contains(errMsg, "NoSuchBucket") { + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在,请检查名称和 Region", cfg.BucketName)}}) + return + } + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: sanitizeStorageCheckError(err)}}) + return + } + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}}) +} + func (h *SystemHandler) ResolveDocumentReader(ctx context.Context, addr string) interfaces.DocumentReader { if addr == "" { return h.documentReader diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index 4abe5943..25ffb149 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -170,7 +170,7 @@ func (c ChunkingConfig) ResolveParserEngine(fileType string) string { // StorageProviderConfig stores the KB-level storage provider selection. // Credentials are managed at the tenant level (StorageEngineConfig). type StorageProviderConfig struct { - Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos", "s3", "oss" + Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos", "s3", "oss", "ks3" } func (c StorageProviderConfig) Value() (driver.Value, error) { @@ -284,7 +284,7 @@ func InferStorageFromFilePath(filePath string) string { // e.g. "minio://bucket/key" → "minio", "local://tenant/file.pdf" → "local" // Returns "" if the path does not use a known provider scheme. func ParseProviderScheme(filePath string) string { - for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss"} { + for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3"} { if strings.HasPrefix(filePath, provider+"://") { return provider } diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 51172dd0..993013ff 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -402,16 +402,17 @@ func (c *ParserEngineConfig) Scan(value interface{}) error { return json.Unmarshal(b, c) } -// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, S3, and OSS. +// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, S3, OSS, and KS3. // Knowledge bases select which provider to use; parameters are read from here. type StorageEngineConfig struct { - DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3", "oss" + DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3", "oss", "ks3" Local *LocalEngineConfig `json:"local,omitempty"` MinIO *MinIOEngineConfig `json:"minio,omitempty"` COS *COSEngineConfig `json:"cos,omitempty"` TOS *TOSEngineConfig `json:"tos,omitempty"` S3 *S3EngineConfig `json:"s3,omitempty"` OSS *OSSEngineConfig `json:"oss,omitempty"` + KS3 *KS3EngineConfig `json:"ks3,omitempty"` } // LocalEngineConfig is for local file system storage (single-machine deployment only). @@ -474,6 +475,16 @@ type OSSEngineConfig struct { TempRegion string `json:"temp_region"` } +// KS3EngineConfig is for Kingsoft Cloud KS3 object storage. +type KS3EngineConfig struct { + Endpoint string `json:"endpoint"` + Region string `json:"region"` + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + BucketName string `json:"bucket_name"` + PathPrefix string `json:"path_prefix"` +} + // Value implements the driver.Valuer interface for StorageEngineConfig func (c *StorageEngineConfig) Value() (driver.Value, error) { if c == nil {