feat(ks3): add KS3 object storage support

Add Kingsoft Cloud KS3 as a supported storage provider, including
  backend service implementation, storage config UI, and provider
  scheme recognition throughout the codebase.
This commit is contained in:
ChenRussell
2026-05-01 19:20:25 +08:00
committed by lyingbug
parent 6f3679716f
commit 42348eb40d
16 changed files with 523 additions and 15 deletions
+11 -2
View File
@@ -169,7 +169,7 @@ export function reconnectDocReader(addr: string): Promise<ParserEnginesResponse
// ---- 存储引擎配置(租户级,供文档/图片存储与 docreader 使用) ----
export interface StorageEngineConfig {
default_provider: string // "local" | "minio" | "cos" | "tos" | "s3" | "oss"
default_provider: string // "local" | "minio" | "cos" | "tos" | "s3" | "oss" | "ks3"
local: { path_prefix: string }
minio: { mode: string; endpoint: string; access_key_id: string; secret_access_key: string; bucket_name: string; use_ssl: boolean; path_prefix: string }
cos: {
@@ -207,6 +207,14 @@ export interface StorageEngineConfig {
temp_bucket_name: string
temp_region: string
}
ks3: {
endpoint: string
region: string
access_key: string
secret_key: string
bucket_name: string
path_prefix: string
}
}
export interface StorageEngineStatusItem {
@@ -235,12 +243,13 @@ export function getStorageEngineStatus(): Promise<{ data: GetStorageEngineStatus
}
export interface StorageCheckRequest {
provider: string // "minio" | "cos" | "tos" | "s3" | "oss"
provider: string // "minio" | "cos" | "tos" | "s3" | "oss" | "ks3"
minio?: StorageEngineConfig['minio']
cos?: StorageEngineConfig['cos']
tos?: StorageEngineConfig['tos']
s3?: StorageEngineConfig['s3']
oss?: StorageEngineConfig['oss']
ks3?: StorageEngineConfig['ks3']
}
export interface StorageCheckResponse {
+10
View File
@@ -754,6 +754,13 @@ export default {
s3Desc: 'AWS S3 and S3-compatible object storage services, suitable for public cloud deployment.',
s3AccessKeyPlaceholder: 'AWS Access Key',
s3SecretKeyPlaceholder: 'AWS Secret Key',
ks3Title: 'Kingsoft Cloud KS3',
ks3Desc: 'Kingsoft Cloud Object Storage Service (KS3), suitable for public cloud deployment.',
ks3AccessKeyPlaceholder: 'Kingsoft Cloud Access Key',
ks3SecretKeyPlaceholder: 'Kingsoft Cloud Secret Key',
ks3EndpointPlaceholder: 'e.g. ks3-cn-beijing.ksyuncs.com',
ks3RegionPlaceholder: 'e.g. BEIJING',
engineKs3: 'Kingsoft Cloud KS3',
ossTitle: 'Alibaba Cloud OSS',
ossDesc: 'Alibaba Cloud Object Storage Service (OSS), suitable for public cloud deployment.',
ossAccessKeyPlaceholder: 'Alibaba Cloud Access Key',
@@ -770,6 +777,7 @@ export default {
cos: 'Tencent Cloud COS',
tos: 'Volcengine TOS',
oss: 'Alibaba Cloud OSS',
ks3: 'Kingsoft Cloud KS3',
},
},
webSearchSettings: {
@@ -3387,6 +3395,8 @@ export default {
engineS3Desc: 'AWS S3 and compatible storage, for public cloud deployment',
engineOss: 'Alibaba Cloud OSS',
engineOssDesc: 'Alibaba Cloud Object Storage Service, for public cloud deployment',
engineKs3: 'Kingsoft Cloud KS3',
engineKs3Desc: 'Kingsoft Cloud Object Storage Service, for public cloud deployment',
},
parser: {
title: 'Parser Engine',
+10
View File
@@ -633,6 +633,13 @@ export default {
s3Desc: 'AWS S3 및 호환 오브젝트 스토리지 서비스, 퍼블릭 클라우드 배포에 적합합니다.',
s3AccessKeyPlaceholder: 'AWS Access Key',
s3SecretKeyPlaceholder: 'AWS Secret Key',
ks3Title: 'Kingsoft Cloud KS3',
ks3Desc: 'Kingsoft Cloud 오브젝트 스토리지 서비스(KS3), 퍼블릭 클라우드 배포에 적합합니다.',
ks3AccessKeyPlaceholder: 'Kingsoft Cloud Access Key',
ks3SecretKeyPlaceholder: 'Kingsoft Cloud Secret Key',
ks3EndpointPlaceholder: 'e.g. ks3-cn-beijing.ksyuncs.com',
ks3RegionPlaceholder: 'e.g. BEIJING',
engineKs3: 'Kingsoft Cloud KS3',
ossTitle: 'Alibaba Cloud OSS',
ossDesc: 'Alibaba Cloud 오브젝트 스토리지 서비스(OSS), 퍼블릭 클라우드 배포에 적합합니다.',
ossAccessKeyPlaceholder: 'Alibaba Cloud Access Key',
@@ -649,6 +656,7 @@ export default {
cos: 'Tencent Cloud COS',
tos: 'Volcengine TOS',
oss: 'Alibaba Cloud OSS',
ks3: 'Kingsoft Cloud KS3',
},
},
webSearchSettings: {
@@ -3455,6 +3463,8 @@ export default {
engineS3Desc: 'AWS S3 및 호환 스토리지, 퍼블릭 클라우드 배포에 적합',
engineOss: 'Alibaba Cloud OSS',
engineOssDesc: 'Alibaba Cloud 오브젝트 스토리지, 퍼블릭 클라우드 배포에 적합',
engineKs3: 'Kingsoft Cloud KS3',
engineKs3Desc: 'Kingsoft Cloud 오브젝트 스토리지, 퍼블릭 클라우드 배포에 적합',
},
parser: {
title: '파서 엔진',
+10
View File
@@ -739,6 +739,13 @@ export default {
s3Desc: 'AWS S3 и совместимые сервисы объектного хранилища для публичного облака.',
s3AccessKeyPlaceholder: 'AWS Access Key',
s3SecretKeyPlaceholder: 'AWS Secret Key',
ks3Title: 'Kingsoft Cloud KS3',
ks3Desc: 'Объектное хранилище Kingsoft Cloud (KS3), подходит для публичного облака.',
ks3AccessKeyPlaceholder: 'Kingsoft Cloud Access Key',
ks3SecretKeyPlaceholder: 'Kingsoft Cloud Secret Key',
ks3EndpointPlaceholder: 'e.g. ks3-cn-beijing.ksyuncs.com',
ks3RegionPlaceholder: 'e.g. BEIJING',
engineKs3: 'Kingsoft Cloud KS3',
ossTitle: 'Alibaba Cloud OSS',
ossDesc: 'Объектное хранилище Alibaba Cloud (OSS), подходит для публичного облака.',
ossAccessKeyPlaceholder: 'Alibaba Cloud Access Key',
@@ -755,6 +762,7 @@ export default {
cos: 'Tencent Cloud COS',
tos: 'Volcengine TOS',
oss: 'Alibaba Cloud OSS',
ks3: 'Kingsoft Cloud KS3',
}
},
webSearchSettings: {
@@ -3064,6 +3072,8 @@ export default {
engineS3Desc: 'AWS S3 и совместимые хранилища, для публичного облака',
engineOss: 'Alibaba Cloud OSS',
engineOssDesc: 'Объектное хранилище Alibaba Cloud, для публичного облака',
engineKs3: 'Kingsoft Cloud KS3',
engineKs3Desc: 'Объектное хранилище Kingsoft Cloud, для публичного облака',
},
parser: {
title: 'Парсер',
+10
View File
@@ -630,6 +630,13 @@ export default {
s3Desc: "AWS S3 及兼容的对象存储服务,适合公有云部署。",
s3AccessKeyPlaceholder: "AWS Access Key",
s3SecretKeyPlaceholder: "AWS Secret Key",
ks3Title: "金山云 KS3",
ks3Desc: "金山云对象存储服务(KS3),适合公有云部署。",
ks3AccessKeyPlaceholder: "金山云 Access Key",
ks3SecretKeyPlaceholder: "金山云 Secret Key",
ks3EndpointPlaceholder: "e.g. ks3-cn-beijing.ksyuncs.com",
ks3RegionPlaceholder: "e.g. BEIJING",
engineKs3: "金山云 KS3",
ossTitle: "阿里云 OSS",
ossDesc: "阿里云对象存储服务(OSS),适合公有云部署。",
ossAccessKeyPlaceholder: "阿里云 Access Key",
@@ -646,6 +653,7 @@ export default {
cos: "腾讯云 COS",
tos: "火山引擎 TOS",
oss: "阿里云 OSS",
ks3: "金山云 KS3",
},
},
webSearchSettings: {
@@ -3384,6 +3392,8 @@ export default {
engineS3Desc: "AWS S3 及兼容存储,适合公有云部署",
engineOss: "阿里云 OSS",
engineOssDesc: "阿里云对象存储,适合公有云部署",
engineKs3: "金山云 KS3",
engineKs3Desc: "金山云对象存储,适合公有云部署",
},
parser: {
title: "解析引擎",
+5 -5
View File
@@ -46,7 +46,7 @@ const DOMPurifyConfig = {
],
USE_PROFILES: { html: true, svg: true, mathMl: true },
// 允许的协议
ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos|s3|oss):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i,
ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos|s3|oss|ks3):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i,
// 禁止的标签和属性
FORBID_TAGS: ['script', 'object', 'embed', 'form', 'input', 'button'],
FORBID_ATTR: ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'],
@@ -124,7 +124,7 @@ function protectProviderImageSrcInHTML(html: string): string {
.replace(/&amp;/g, '&')
.replace(/&quot;/g, '"');
return html.replace(
/<img\b([^>]*?)\ssrc=(["'])(local|minio|cos|tos|s3|oss):(?:\/\/|&#x2f;&#x2f;|&#47;&#47;)([^"']+)\2([^>]*)>/gi,
/<img\b([^>]*?)\ssrc=(["'])(local|minio|cos|tos|s3|oss|ks3):(?:\/\/|&#x2f;&#x2f;|&#47;&#47;)([^"']+)\2([^>]*)>/gi,
(_m, before, quote, provider, restPathRaw, after) => {
const restPath = decodeProviderURL(restPathRaw);
const protectedSrc = `${provider}://${restPath}`;
@@ -178,7 +178,7 @@ export function isValidURL(url: string): boolean {
}
// 允许 provider:// 形式,由前端后续鉴权拉取并替换为 blob URL
if (/^(local|minio|cos|tos|s3|oss):\/\/\S+$/i.test(trimmed)) {
if (/^(local|minio|cos|tos|s3|oss|ks3):\/\/\S+$/i.test(trimmed)) {
return true;
}
@@ -304,7 +304,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef
}
const images = root.querySelectorAll<HTMLImageElement>(
'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"]',
'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"], img[src^="ks3://"]',
);
if (!images.length) {
return;
@@ -324,7 +324,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef
}
img.dataset.authHydrated = '1';
const isProviderScheme = /^(local|minio|cos|tos|s3|oss):\/\//.test(sourceURL);
const isProviderScheme = /^(local|minio|cos|tos|s3|oss|ks3):\/\//.test(sourceURL);
const requestURL = isProviderScheme
? `/files?${new URLSearchParams({ file_path: sourceURL }).toString()}`
: sourceURL;
@@ -131,6 +131,14 @@ const engineOptions = computed(() => {
available: statusMap.oss,
disabled: allowedMap.oss === false || statusMap.oss === false,
},
{
value: 'ks3',
label: t('kbSettings.storage.engineKs3'),
desc: t('kbSettings.storage.engineKs3Desc'),
allowed: allowedMap.ks3 !== false,
available: statusMap.ks3,
disabled: allowedMap.ks3 === false || statusMap.ks3 === false,
},
]
})
@@ -171,7 +179,7 @@ async function load() {
allowedProviders.value = statusRes?.data?.allowed_providers ?? []
defaultProvider.value = configRes?.data?.default_provider || 'local'
const d = configRes?.data
hasAnyConfig.value = !!(d?.local?.path_prefix || d?.minio?.bucket_name || d?.cos?.bucket_name || d?.tos?.bucket_name || d?.s3?.bucket_name)
hasAnyConfig.value = !!(d?.local?.path_prefix || d?.minio?.bucket_name || d?.cos?.bucket_name || d?.tos?.bucket_name || d?.s3?.bucket_name || d?.oss?.bucket_name || d?.ks3?.bucket_name)
if (!localProvider.value || localProvider.value === '') {
localProvider.value = defaultProvider.value
emit('update:storageProvider', localProvider.value)
@@ -123,6 +123,18 @@
</div>
<p class="engine-card-desc">{{ $t('settings.storage.ossDesc') }}</p>
</div>
<div
v-if="isProviderAllowed('ks3')"
:class="['engine-card', { active: drawerVisible && currentEngine === 'ks3' }]"
@click="openDrawer('ks3')"
>
<div class="engine-card-header">
<h3>{{ $t('settings.storage.ks3Title') }}</h3>
<t-tag theme="success" variant="light" size="small">{{ $t('settings.storage.configurable') }}</t-tag>
</div>
<p class="engine-card-desc">{{ $t('settings.storage.ks3Desc') }}</p>
</div>
</div>
</template>
@@ -378,6 +390,65 @@
</div>
</template>
<template v-else-if="currentEngine === 'ks3'">
<div class="engine-info-block">
<p class="engine-desc">
{{ $t('settings.storage.ks3Desc') }}
</p>
</div>
<div class="engine-form">
<div class="form-item">
<label class="form-label">Endpoint</label>
<t-input
v-model="config.ks3.endpoint"
:placeholder="$t('settings.storage.ks3EndpointPlaceholder')"
clearable
/>
</div>
<div class="form-item">
<label class="form-label">Region</label>
<t-input
v-model="config.ks3.region"
:placeholder="$t('settings.storage.ks3RegionPlaceholder')"
clearable
/>
</div>
<div class="form-item">
<label class="form-label">Access Key</label>
<t-input
v-model="config.ks3.access_key"
:placeholder="$t('settings.storage.ks3AccessKeyPlaceholder')"
clearable
/>
</div>
<div class="form-item">
<label class="form-label">Secret Key</label>
<t-input
v-model="config.ks3.secret_key"
type="password"
:placeholder="$t('settings.storage.ks3SecretKeyPlaceholder')"
clearable
/>
</div>
<div class="form-item">
<label class="form-label">{{ $t('settings.storage.bucketName') }}</label>
<t-input
v-model="config.ks3.bucket_name"
:placeholder="$t('settings.storage.bucketPlaceholder')"
clearable
/>
</div>
<div class="form-item">
<label class="form-label">{{ $t('settings.storage.pathPrefix') }}</label>
<t-input
v-model="config.ks3.path_prefix"
:placeholder="$t('settings.storage.prefixPlaceholder')"
clearable
/>
</div>
</div>
</template>
<div class="form-item" v-if="currentEngine && currentEngine !== 'local'">
<label class="form-label">{{ $t('settings.storage.testConnection') }}</label>
<div class="api-test-section">
@@ -432,6 +503,14 @@ const defaultConfig = (): StorageEngineConfig => ({
temp_bucket_name: '',
temp_region: '',
},
ks3: {
endpoint: '',
region: '',
access_key: '',
secret_key: '',
bucket_name: '',
path_prefix: '',
},
})
const loading = ref(true)
@@ -454,6 +533,8 @@ const checkingS3 = ref(false)
const s3CheckResult = ref<{ ok: boolean; message: string } | null>(null)
const checkingOss = ref(false)
const ossCheckResult = ref<{ ok: boolean; message: string } | null>(null)
const checkingKs3 = ref(false)
const ks3CheckResult = ref<{ ok: boolean; message: string } | null>(null)
const drawerVisible = ref(false)
const currentEngine = ref<string | null>(null)
@@ -465,6 +546,7 @@ const providerOptions = computed(() => [
{ value: 'tos', label: t('settings.storage.engineTos'), allowed: isProviderAllowed('tos') },
{ value: 's3', label: 'AWS S3', allowed: isProviderAllowed('s3') },
{ value: 'oss', label: t('settings.storage.engineOss'), allowed: isProviderAllowed('oss') },
{ value: 'ks3', label: t('settings.storage.engineKs3'), allowed: isProviderAllowed('ks3') },
])
const hasAllowedProviders = computed(() => (allowedProviders.value?.length ?? 0) > 0)
@@ -481,6 +563,8 @@ const currentCheckState = computed(() => {
return { loading: checkingS3.value, result: s3CheckResult.value, onCheck: onCheckS3 }
case 'oss':
return { loading: checkingOss.value, result: ossCheckResult.value, onCheck: onCheckOss }
case 'ks3':
return { loading: checkingKs3.value, result: ks3CheckResult.value, onCheck: onCheckKs3 }
default:
return { loading: false, result: null, onCheck: () => undefined }
}
@@ -495,6 +579,7 @@ const drawerTitle = computed(() => {
tos: t('settings.storage.tosTitle'),
s3: t('settings.storage.s3Title'),
oss: t('settings.storage.ossTitle'),
ks3: t('settings.storage.ks3Title'),
}
return titles[currentEngine.value] || currentEngine.value
})
@@ -526,6 +611,7 @@ function openDrawer(engine: string) {
tosCheckResult.value = null
s3CheckResult.value = null
ossCheckResult.value = null
ks3CheckResult.value = null
}
async function loadConfig() {
@@ -590,6 +676,16 @@ async function loadConfig() {
temp_region: d.oss.temp_region || '',
}
: defaultConfig().oss,
ks3: d.ks3
? {
endpoint: d.ks3.endpoint || '',
region: d.ks3.region || '',
access_key: d.ks3.access_key || '',
secret_key: d.ks3.secret_key || '',
bucket_name: d.ks3.bucket_name || '',
path_prefix: d.ks3.path_prefix || '',
}
: defaultConfig().ks3,
}
}
} catch {
@@ -681,6 +777,14 @@ function buildPayload(): StorageEngineConfig {
temp_bucket_name: (config.value.oss?.temp_bucket_name || '').trim(),
temp_region: (config.value.oss?.temp_region || '').trim(),
},
ks3: {
endpoint: (config.value.ks3?.endpoint || '').trim(),
region: (config.value.ks3?.region || '').trim(),
access_key: (config.value.ks3?.access_key || '').trim(),
secret_key: (config.value.ks3?.secret_key || '').trim(),
bucket_name: (config.value.ks3?.bucket_name || '').trim(),
path_prefix: (config.value.ks3?.path_prefix || '').trim(),
},
}
}
@@ -777,6 +881,20 @@ async function onCheckOss() {
}
}
async function onCheckKs3() {
checkingKs3.value = true
ks3CheckResult.value = null
try {
const payload = buildPayload()
const res = await checkStorageEngine({ provider: 'ks3', ks3: payload.ks3 })
ks3CheckResult.value = res?.data ?? { ok: false, message: t('settings.storage.unknownError') }
} catch (e: unknown) {
ks3CheckResult.value = { ok: false, message: e instanceof Error ? e.message : t('settings.storage.requestFailed') }
} finally {
checkingKs3.value = false
}
}
onMounted(loadAll)
</script>
+1
View File
@@ -204,6 +204,7 @@ require (
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/ks3sdklib/aws-sdk-go v1.11.0 // indirect
github.com/labstack/echo/v4 v4.13.3 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/leaanthony/go-ansi-parser v1.6.1 // indirect
+3
View File
@@ -2099,6 +2099,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ks3sdklib/aws-sdk-go v1.11.0 h1:CTYXt7KzMdbm59h0a3DIWZN1jN2Qi5SP5OjCyWX01vE=
github.com/ks3sdklib/aws-sdk-go v1.11.0/go.mod h1:fOYkqRwykBT2tHD/UkZ2qHKxEtKNJd2gJVt2fveKRk0=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
@@ -2718,6 +2720,7 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
@@ -126,6 +126,17 @@ func NewFileServiceFromStorageConfig(
}
return svc, p, err
case "ks3":
if sec == nil || sec.KS3 == nil || sec.KS3.Endpoint == "" || sec.KS3.Region == "" || sec.KS3.AccessKey == "" || sec.KS3.SecretKey == "" || sec.KS3.BucketName == "" {
return nil, p, fmt.Errorf("incomplete ks3 config")
}
pathPrefix := strings.TrimSpace(sec.KS3.PathPrefix)
if pathPrefix == "" {
pathPrefix = "weknora/"
}
svc, err := NewKS3FileService(sec.KS3.Endpoint, sec.KS3.Region, sec.KS3.AccessKey, sec.KS3.SecretKey, sec.KS3.BucketName, pathPrefix)
return svc, p, err
default:
return nil, p, fmt.Errorf("unsupported provider %q", p)
}
+255
View File
@@ -0,0 +1,255 @@
package file
import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
ks3aws "github.com/ks3sdklib/aws-sdk-go/aws"
"github.com/ks3sdklib/aws-sdk-go/aws/credentials"
ks3s3 "github.com/ks3sdklib/aws-sdk-go/service/s3"
)
const ks3Scheme = "ks3://"
// ks3FileService implements FileService for Kingsoft Cloud KS3.
// KS3 uses V2 signing by default and virtual-hosted style addressing,
// so it cannot be handled by the generic S3 provider without workarounds.
type ks3FileService struct {
client *ks3s3.S3
bucketName string
pathPrefix string
}
// NewKS3FileService creates a KS3 file service and ensures the bucket exists.
func NewKS3FileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) {
client, err := newKS3Client(endpoint, region, accessKey, secretKey)
if err != nil {
return nil, err
}
pathPrefix = strings.Trim(pathPrefix, "/")
svc := &ks3FileService{
client: client,
bucketName: bucketName,
pathPrefix: pathPrefix,
}
if err := ensureKS3Bucket(client, bucketName); err != nil {
return nil, err
}
return svc, nil
}
func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, error) {
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
client := ks3s3.New(&ks3aws.Config{
Credentials: creds,
Region: region,
Endpoint: endpoint,
DisableSSL: false,
S3ForcePathStyle: false, // KS3 uses virtual-hosted style
SignerVersion: "V2", // KS3 recommends V2 signing
MaxRetries: 3,
})
return client, nil
}
func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error {
_, err := client.HeadBucket(&ks3s3.HeadBucketInput{
Bucket: ks3aws.String(bucketName),
})
if err == nil {
return nil
}
// Bucket doesn't exist, try to create it
_, createErr := client.CreateBucket(&ks3s3.CreateBucketInput{
Bucket: ks3aws.String(bucketName),
})
if createErr != nil {
return fmt.Errorf("failed to create KS3 bucket %q: %w", bucketName, createErr)
}
return nil
}
// CheckKS3Connectivity tests KS3 connectivity using the provided credentials.
func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
client, err := newKS3Client(endpoint, region, accessKey, secretKey)
if err != nil {
return err
}
done := make(chan error, 1)
go func() {
_, err := client.HeadBucket(&ks3s3.HeadBucketInput{
Bucket: ks3aws.String(bucketName),
})
done <- err
}()
select {
case <-ctx.Done():
return ctx.Err()
case err := <-done:
return err
}
}
func joinKS3Key(parts ...string) string {
filtered := make([]string, 0, len(parts))
for _, p := range parts {
p = strings.Trim(p, "/")
if p != "" {
filtered = append(filtered, p)
}
}
return strings.Join(filtered, "/")
}
func parseKS3FilePath(filePath string) (bucket, objectKey string, err error) {
if !strings.HasPrefix(filePath, ks3Scheme) {
return "", "", fmt.Errorf("invalid KS3 file path: %s", filePath)
}
rest := strings.TrimPrefix(filePath, ks3Scheme)
parts := strings.SplitN(rest, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return "", "", fmt.Errorf("invalid KS3 file path: %s", filePath)
}
return parts[0], parts[1], nil
}
func (s *ks3FileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
ext := filepath.Ext(file.Filename)
objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), knowledgeID, uuid.New().String()+ext)
src, err := file.Open()
if err != nil {
return "", fmt.Errorf("failed to open file: %w", err)
}
defer src.Close()
contentType := file.Header.Get("Content-Type")
if contentType == "" {
contentType = utils.GetContentTypeByExt(ext)
}
_, err = s.client.PutObject(&ks3s3.PutObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
Body: src,
ContentType: ks3aws.String(contentType),
})
if err != nil {
return "", fmt.Errorf("failed to upload file to KS3: %w", err)
}
return fmt.Sprintf("%s%s/%s", ks3Scheme, s.bucketName, objectKey), nil
}
func (s *ks3FileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
safeName, err := utils.SafeFileName(fileName)
if err != nil {
return "", fmt.Errorf("invalid file name: %w", err)
}
ext := filepath.Ext(safeName)
objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), "exports", uuid.New().String()+ext)
_, err = s.client.PutObject(&ks3s3.PutObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
Body: bytes.NewReader(data),
ContentType: ks3aws.String(utils.GetContentTypeByExt(ext)),
})
if err != nil {
return "", fmt.Errorf("failed to upload bytes to KS3: %w", err)
}
return fmt.Sprintf("%s%s/%s", ks3Scheme, s.bucketName, objectKey), nil
}
func (s *ks3FileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
_, objectKey, err := parseKS3FilePath(filePath)
if err != nil {
return nil, err
}
if err := utils.SafeObjectKey(objectKey); err != nil {
return nil, fmt.Errorf("invalid file path: %w", err)
}
resp, err := s.client.GetObject(&ks3s3.GetObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
})
if err != nil {
return nil, fmt.Errorf("failed to get file from KS3: %w", err)
}
return resp.Body, nil
}
func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error {
_, objectKey, err := parseKS3FilePath(filePath)
if err != nil {
return err
}
if err := utils.SafeObjectKey(objectKey); err != nil {
return fmt.Errorf("invalid file path: %w", err)
}
_, err = s.client.DeleteObject(&ks3s3.DeleteObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
})
if err != nil {
return fmt.Errorf("failed to delete file from KS3: %w", err)
}
return nil
}
func (s *ks3FileService) CheckConnectivity(ctx context.Context) error {
done := make(chan error, 1)
go func() {
_, err := s.client.HeadBucket(&ks3s3.HeadBucketInput{
Bucket: ks3aws.String(s.bucketName),
})
done <- err
}()
select {
case <-ctx.Done():
return ctx.Err()
case err := <-done:
return err
}
}
func (s *ks3FileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
_, objectKey, err := parseKS3FilePath(filePath)
if err != nil {
return "", err
}
if err := utils.SafeObjectKey(objectKey); err != nil {
return "", fmt.Errorf("invalid file path: %w", err)
}
url, err := s.client.GeneratePresignedUrl(&ks3s3.GeneratePresignedUrlInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
HTTPMethod: ks3s3.HTTPMethod("GET"),
Expires: int64((24 * time.Hour).Seconds()),
})
if err != nil {
return "", fmt.Errorf("failed to generate KS3 presigned URL: %w", err)
}
return url, nil
}
+1 -1
View File
@@ -7,7 +7,7 @@ import (
const storageAllowListEnv = "STORAGE_ALLOW_LIST"
var supportedStorageProviders = []string{"local", "minio", "cos", "tos", "s3", "oss"}
var supportedStorageProviders = []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3"}
func getSupportedStorageProviders() []string {
providers := make([]string, len(supportedStorageProviders))
+54 -2
View File
@@ -432,6 +432,17 @@ func (h *SystemHandler) isOSSConfigured(c *gin.Context) bool {
return false
}
// isKS3Configured checks whether KS3 connection info is available from tenant config.
func (h *SystemHandler) isKS3Configured(c *gin.Context) bool {
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.KS3 != nil {
ks3Conf := tenant.StorageEngineConfig.KS3
return ks3Conf.Endpoint != "" && ks3Conf.Region != "" && ks3Conf.AccessKey != "" && ks3Conf.SecretKey != "" && ks3Conf.BucketName != ""
}
}
return false
}
// isTOSEnvAvailable checks whether TOS env vars are set.
func (h *SystemHandler) isTOSEnvAvailable() bool {
return os.Getenv("TOS_ENDPOINT") != "" &&
@@ -443,7 +454,7 @@ func (h *SystemHandler) isTOSEnvAvailable() bool {
// StorageEngineStatusItem describes one storage engine's availability and description.
type StorageEngineStatusItem struct {
Name string `json:"name"` // "local", "minio", "cos", "tos"
Name string `json:"name"` // "local", "minio", "cos", "tos"
Allowed bool `json:"allowed"`
Available bool `json:"available"` // whether the engine can be used
Description string `json:"description"` // short description for UI
@@ -470,6 +481,7 @@ func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) {
tosConfigured := h.isTOSConfigured(c)
s3Configured := h.isS3Configured(c)
ossConfigured := h.isOSSConfigured(c)
ks3Configured := h.isKS3Configured(c)
allowed := getAllowedStorageProviders()
allowedProviders := make([]string, 0, len(supportedStorageProviders))
for _, provider := range getSupportedStorageProviders() {
@@ -484,6 +496,7 @@ func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) {
{Name: "tos", Allowed: allowed["tos"], Available: tosConfigured, Description: "火山引擎对象存储服务,适合公有云部署"},
{Name: "s3", Allowed: allowed["s3"], Available: s3Configured, Description: "AWS S3 与兼容对象存储服务,适合公有云与混合云部署"},
{Name: "oss", Allowed: allowed["oss"], Available: ossConfigured, Description: "阿里云对象存储服务,适合公有云部署,支持 S3 兼容协议"},
{Name: "ks3", Allowed: allowed["ks3"], Available: ks3Configured, Description: "金山云对象存储服务,适合公有云部署"},
}
c.JSON(200, gin.H{
"code": 0,
@@ -586,12 +599,13 @@ func isBlockedStorageEndpoint(endpoint string) (bool, string) {
// StorageCheckRequest is the body for POST /system/storage-engine-check.
type StorageCheckRequest struct {
Provider string `json:"provider"` // "minio", "cos", "tos", "s3", "oss"
Provider string `json:"provider"` // "minio", "cos", "tos", "s3", "oss", "ks3"
MinIO *types.MinIOEngineConfig `json:"minio,omitempty"`
COS *types.COSEngineConfig `json:"cos,omitempty"`
TOS *types.TOSEngineConfig `json:"tos,omitempty"`
S3 *types.S3EngineConfig `json:"s3,omitempty"`
OSS *types.OSSEngineConfig `json:"oss,omitempty"`
KS3 *types.KS3EngineConfig `json:"ks3,omitempty"`
}
// StorageCheckResponse is the response for a single-engine connectivity check.
@@ -634,6 +648,8 @@ func (h *SystemHandler) CheckStorageEngine(c *gin.Context) {
h.checkS3(c, ctx, req.S3)
case "oss":
h.checkOSS(c, ctx, req.OSS)
case "ks3":
h.checkKS3(c, ctx, req.KS3)
default:
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: "本地存储无需检测"}})
}
@@ -867,6 +883,42 @@ func (h *SystemHandler) checkOSS(c *gin.Context, ctx context.Context, cfg *types
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}})
}
func (h *SystemHandler) checkKS3(c *gin.Context, ctx context.Context, cfg *types.KS3EngineConfig) {
if cfg == nil {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 KS3 配置"}})
return
}
endpoint, region, accessKey, secretKey := cfg.Endpoint, cfg.Region, cfg.AccessKey, cfg.SecretKey
if endpoint == "" || region == "" || accessKey == "" || secretKey == "" || cfg.BucketName == "" {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Endpoint、Region、Access Key、Secret Key、Bucket 名称不能为空"}})
return
}
if blocked, reason := isBlockedStorageEndpoint(endpoint); blocked {
logger.Warnf(ctx, "Storage check: KS3 endpoint blocked by SSRF protection, endpoint: %s", endpoint)
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}})
return
}
err := file.CheckKS3Connectivity(ctx, endpoint, region, accessKey, secretKey, cfg.BucketName)
if err != nil {
logger.Errorf(ctx, "Storage check: KS3 connectivity failed, bucket: %s, error: %v", cfg.BucketName, err)
errMsg := err.Error()
if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "AccessDenied") {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "认证失败,请检查 Access Key / Secret Key 是否正确"}})
return
}
if strings.Contains(errMsg, "404") || strings.Contains(errMsg, "NoSuchBucket") {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在,请检查名称和 Region", cfg.BucketName)}})
return
}
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: sanitizeStorageCheckError(err)}})
return
}
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}})
}
func (h *SystemHandler) ResolveDocumentReader(ctx context.Context, addr string) interfaces.DocumentReader {
if addr == "" {
return h.documentReader
+2 -2
View File
@@ -170,7 +170,7 @@ func (c ChunkingConfig) ResolveParserEngine(fileType string) string {
// StorageProviderConfig stores the KB-level storage provider selection.
// Credentials are managed at the tenant level (StorageEngineConfig).
type StorageProviderConfig struct {
Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos", "s3", "oss"
Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos", "s3", "oss", "ks3"
}
func (c StorageProviderConfig) Value() (driver.Value, error) {
@@ -284,7 +284,7 @@ func InferStorageFromFilePath(filePath string) string {
// e.g. "minio://bucket/key" → "minio", "local://tenant/file.pdf" → "local"
// Returns "" if the path does not use a known provider scheme.
func ParseProviderScheme(filePath string) string {
for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss"} {
for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3"} {
if strings.HasPrefix(filePath, provider+"://") {
return provider
}
+13 -2
View File
@@ -402,16 +402,17 @@ func (c *ParserEngineConfig) Scan(value interface{}) error {
return json.Unmarshal(b, c)
}
// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, S3, and OSS.
// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, S3, OSS, and KS3.
// Knowledge bases select which provider to use; parameters are read from here.
type StorageEngineConfig struct {
DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3", "oss"
DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3", "oss", "ks3"
Local *LocalEngineConfig `json:"local,omitempty"`
MinIO *MinIOEngineConfig `json:"minio,omitempty"`
COS *COSEngineConfig `json:"cos,omitempty"`
TOS *TOSEngineConfig `json:"tos,omitempty"`
S3 *S3EngineConfig `json:"s3,omitempty"`
OSS *OSSEngineConfig `json:"oss,omitempty"`
KS3 *KS3EngineConfig `json:"ks3,omitempty"`
}
// LocalEngineConfig is for local file system storage (single-machine deployment only).
@@ -474,6 +475,16 @@ type OSSEngineConfig struct {
TempRegion string `json:"temp_region"`
}
// KS3EngineConfig is for Kingsoft Cloud KS3 object storage.
type KS3EngineConfig struct {
Endpoint string `json:"endpoint"`
Region string `json:"region"`
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
BucketName string `json:"bucket_name"`
PathPrefix string `json:"path_prefix"`
}
// Value implements the driver.Valuer interface for StorageEngineConfig
func (c *StorageEngineConfig) Value() (driver.Value, error) {
if c == nil {