feat(storage): add multi-instance storage backends per workspace

Introduce tenant-scoped storage backend registry so workspaces can register
multiple storage instances and bind each knowledge base to a specific backend,
with migration from legacy tenant-level storage config and new settings UI.
This commit is contained in:
wizardchen
2026-07-15 13:26:29 +08:00
committed by lyingbug
parent 139ec1e6b8
commit f8b3e5f148
58 changed files with 2863 additions and 470 deletions
+1
View File
@@ -142,6 +142,7 @@ export interface KBModelConfigRequest {
enabled: boolean
}
/** 存储引擎选择:"local" | "minio" | "cos" | "obs" 等,影响文档上传与文档内图片存储 */
storageBackendId?: string
storageProvider?: string
nodeExtract: {
enabled: boolean
+3
View File
@@ -65,6 +65,9 @@ export function createKnowledgeBase(data: {
// store. Immutable after creation — UpdateKnowledgeBase intentionally
// does not accept this field.
vector_store_id?: string;
// Concrete tenant-owned storage instance. When omitted, the tenant default
// backend is bound by the server at creation time.
storage_backend_id?: string;
vlm_config?: {
enabled: boolean;
model_id?: string;
+45
View File
@@ -0,0 +1,45 @@
import { get, post, put, del } from '@/utils/request'
export interface StorageBackendConfig {
mode?: string
endpoint?: string
region?: string
access_key_id?: string
secret_access_key?: string
bucket_name?: string
path_prefix?: string
app_id?: string
use_ssl?: boolean
force_path_style?: boolean
use_temp_bucket?: boolean
temp_bucket_name?: string
temp_region?: string
}
export interface StorageBackend {
id: string
tenant_id?: number
name: string
provider: string
config: StorageBackendConfig
source: 'user' | 'env'
status: 'active' | 'disabled'
legacy_alias?: boolean
created_at?: string
updated_at?: string
}
export interface StorageBackendListResponse {
success: boolean
data: StorageBackend[]
default_storage_backend_id?: string | null
}
export const listStorageBackends = (): Promise<StorageBackendListResponse> => get('/api/v1/storage-backends')
export const listStorageBackendTypes = (): Promise<{ success: boolean; data: string[] }> => get('/api/v1/storage-backends/types')
export const createStorageBackend = (data: Partial<StorageBackend>) => post('/api/v1/storage-backends', data)
export const updateStorageBackend = (id: string, data: Partial<StorageBackend>) => put(`/api/v1/storage-backends/${id}`, data)
export const deleteStorageBackend = (id: string) => del(`/api/v1/storage-backends/${id}`)
export const setDefaultStorageBackend = (id: string) => put(`/api/v1/storage-backends/${id}/default`, {})
export const testStorageBackend = (data: Partial<StorageBackend>) => post('/api/v1/storage-backends/test', data)
export const testStorageBackendByID = (id: string) => post(`/api/v1/storage-backends/${id}/test`, {})
+1
View File
@@ -79,6 +79,7 @@ export type TenantAPIKeyCapability =
| 'manage_datasources'
| 'manage_channels'
| 'manage_vector_stores'
| 'manage_storage_backends'
| 'manage_web_search'
| 'run_evaluations'
| 'manage_members'
@@ -0,0 +1,9 @@
<svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
<title>Local storage</title>
<rect x="3" y="4" width="18" height="7" rx="2" fill="#5C6678"/>
<rect x="3" y="13" width="18" height="7" rx="2" fill="#434C5E"/>
<rect x="5.5" y="7.1" width="13" height="0.8" rx="0.4" fill="#8A94A6"/>
<rect x="5.5" y="16.1" width="13" height="0.8" rx="0.4" fill="#8A94A6"/>
<circle cx="7.2" cy="7.5" r="0.9" fill="#6B7280"/>
<circle cx="7.2" cy="16.5" r="0.9" fill="#6B7280"/>
</svg>

After

Width:  |  Height:  |  Size: 499 B

+2
View File
@@ -6240,6 +6240,8 @@ export default {
capabilityManageChannelsHint: 'Manage agent embed channels, IM channels, and WeChat QR binding flows.',
capabilityManageVectorStores: 'Manage retrieval infrastructure',
capabilityManageVectorStoresHint: 'Manage vector-store configuration plus parser, document reader, and storage engine connectivity checks.',
capabilityManageStorageBackends: 'Manage storage backends',
capabilityManageStorageBackendsHint: 'Manage object/file storage backend instances (e.g. S3-compatible or local file storage): their CRUD lifecycle, connectivity tests, and the tenant default selection.',
capabilityManageWebSearch: 'Manage web search',
capabilityManageWebSearchHint: 'Manage web-search provider configurations, credentials, and connection tests.',
capabilityRunEvaluations: 'Run evaluations',
+2
View File
@@ -6244,6 +6244,8 @@ export default {
capabilityManageChannelsHint: "에이전트 임베드 채널, IM 채널 및 WeChat QR 바인딩 흐름을 관리합니다.",
capabilityManageVectorStores: "검색 인프라 관리",
capabilityManageVectorStoresHint: "벡터 스토어 설정과 파서, 문서 리더, 스토리지 엔진 연결 검사를 관리합니다.",
capabilityManageStorageBackends: "스토리지 백엔드 관리",
capabilityManageStorageBackendsHint: "객체/파일 스토리지 백엔드 인스턴스(예: S3 호환 또는 로컬 파일 스토리지)의 CRUD, 연결 테스트 및 테넌트 기본 스토리지 설정을 관리합니다.",
capabilityManageWebSearch: "웹 검색 관리",
capabilityManageWebSearchHint: "웹 검색 공급자 설정, 자격 증명 및 연결 테스트를 관리합니다.",
capabilityRunEvaluations: "평가 실행",
+2
View File
@@ -6065,6 +6065,8 @@ export default {
capabilityManageChannelsHint: 'Управление embed-каналами агентов, IM-каналами и привязкой WeChat через QR-код.',
capabilityManageVectorStores: 'Управление инфраструктурой поиска',
capabilityManageVectorStoresHint: 'Управление векторными хранилищами и проверками парсеров, чтения документов и storage engine.',
capabilityManageStorageBackends: 'Управление хранилищами (backends)',
capabilityManageStorageBackendsHint: 'Управление экземплярами объектных/файловых хранилищ (например, S3-совместимых или локальных): их CRUD, проверки подключения и выбор хранилища по умолчанию для арендатора.',
capabilityManageWebSearch: 'Управление веб-поиском',
capabilityManageWebSearchHint: 'Управление провайдерами веб-поиска, учётными данными и проверками подключения.',
capabilityRunEvaluations: 'Запуск оценок',
+2
View File
@@ -6265,6 +6265,8 @@ export default {
capabilityManageChannelsHint: "允许管理智能体的嵌入渠道、IM 渠道和微信扫码绑定流程。",
capabilityManageVectorStores: "管理检索基础设施",
capabilityManageVectorStoresHint: "允许管理向量库配置,以及解析器、文档读取器和存储引擎连通性检查。",
capabilityManageStorageBackends: "管理存储后端",
capabilityManageStorageBackendsHint: "允许管理对象/文件存储后端实例(如 S3 兼容或本地文件存储)的增删改查、连通性测试及租户默认存储设置。",
capabilityManageWebSearch: "管理联网搜索",
capabilityManageWebSearchHint: "允许管理联网搜索供应商配置、凭据和连接测试。",
capabilityRunEvaluations: "运行评测",
@@ -645,6 +645,7 @@ const API_KEY_CAPABILITIES: TenantAPIKeyCapability[] = [
'manage_datasources',
'manage_models',
'manage_vector_stores',
'manage_storage_backends',
'manage_web_search',
'manage_channels',
'run_evaluations',
@@ -699,6 +700,7 @@ const apiKeyCapabilityGroups: ApiKeyCapabilityGroup[] = [
capabilities: [
{ value: 'manage_models', labelKey: 'integrations.api.capabilityManageModels', hintKey: 'integrations.api.capabilityManageModelsHint' },
{ value: 'manage_vector_stores', labelKey: 'integrations.api.capabilityManageVectorStores', hintKey: 'integrations.api.capabilityManageVectorStoresHint' },
{ value: 'manage_storage_backends', labelKey: 'integrations.api.capabilityManageStorageBackends', hintKey: 'integrations.api.capabilityManageStorageBackendsHint' },
{ value: 'manage_web_search', labelKey: 'integrations.api.capabilityManageWebSearch', hintKey: 'integrations.api.capabilityManageWebSearchHint' },
{ value: 'manage_channels', labelKey: 'integrations.api.capabilityManageChannels', hintKey: 'integrations.api.capabilityManageChannelsHint' },
{ value: 'run_evaluations', labelKey: 'integrations.api.capabilityRunEvaluations', hintKey: 'integrations.api.capabilityRunEvaluationsHint' },
@@ -786,6 +788,7 @@ function keyCapabilityLabels(key: TenantAPIKey): string[] {
manage_datasources: t('integrations.api.capabilityManageDatasources'),
manage_channels: t('integrations.api.capabilityManageChannels'),
manage_vector_stores: t('integrations.api.capabilityManageVectorStores'),
manage_storage_backends: t('integrations.api.capabilityManageStorageBackends'),
manage_web_search: t('integrations.api.capabilityManageWebSearch'),
run_evaluations: t('integrations.api.capabilityRunEvaluations'),
manage_members: t('integrations.api.capabilityManageMembers'),
@@ -254,8 +254,10 @@
<!-- 存储引擎 -->
<div v-if="!isFAQ && formData && currentSection === 'storage'" class="section">
<KBStorageSettings
:storage-backend-id="formData.storageBackendId"
:storage-provider="formData.storageProvider"
:has-files="mode === 'edit' && hasFiles"
@update:storage-backend-id="handleStorageBackendUpdate"
@update:storage-provider="handleStorageProviderUpdate"
/>
</div>
@@ -711,6 +713,7 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
languages: [] as string[],
tableMetadataInstructions: ''
},
storageBackendId: '' as string,
storageProvider: '' as string,
multimodalConfig: {
enabled: false,
@@ -832,6 +835,7 @@ const loadKBData = async () => {
languages: kb.chunking_config?.languages || [],
tableMetadataInstructions: kb.chunking_config?.table_metadata_instructions || ''
},
storageBackendId: (kb.storage_backend_id || '') as string,
storageProvider: (kb.storage_provider_config?.provider || kb.storage_config?.provider || 'local') as string,
multimodalConfig: {
enabled: !!kb.vlm_config?.enabled,
@@ -1025,6 +1029,12 @@ const handleStorageProviderUpdate = (value: string) => {
}
}
const handleStorageBackendUpdate = (value: string) => {
if (formData.value) {
formData.value.storageBackendId = value
}
}
async function loadTenantDefaultStorageProvider(force = false) {
try {
await editorResources.ensureStorageEngine(force)
@@ -1177,8 +1187,11 @@ const buildSubmitData = () => {
language: formData.value.asrConfig?.language || ''
}
// 存储引擎:仅传 provider,参数从全局设置读取
// Write to storage_provider_config (authoritative) + storage_config (legacy dual-write)
// storage_backend_id is authoritative. Keep provider projection for old clients
// and for rolling upgrades where a node has not picked up the new schema yet.
if (formData.value.storageBackendId) {
data.storage_backend_id = formData.value.storageBackendId
}
const storageProvider = resolvedStorageProvider()
data.storage_provider_config = {
provider: storageProvider
@@ -1362,6 +1375,7 @@ const doSubmit = async () => {
multimodal: {
enabled: !!data.vlm_config?.enabled
},
storageBackendId: formData.value?.storageBackendId || '',
storageProvider: data.storage_provider_config?.provider || data.storage_config?.provider || 'local',
nodeExtract: {
enabled: data.extract_config?.enabled || false,
@@ -2,48 +2,28 @@
<div class="kb-storage-settings">
<div class="section-header">
<h2>{{ $t('kbSettings.storage.title') }}</h2>
<p class="section-description">
{{ $t('kbSettings.storage.description') }}
</p>
<p class="section-description">选择此知识库绑定的具体存储实例。</p>
</div>
<div v-if="loading" class="loading-inline">
<t-loading size="small" />
<span>{{ $t('kbSettings.storage.loading') }}</span>
</div>
<div v-if="loading" class="loading-inline"><t-loading size="small" /><span>加载中...</span></div>
<div v-else class="settings-group">
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('kbSettings.storage.engineLabel') }}</label>
<p class="desc">{{ $t('kbSettings.storage.engineDesc') }}</p>
<label>存储实例</label>
<p class="desc">同一种存储类型可以配置多个不同 Endpoint、Bucket 或凭据的实例。</p>
</div>
<div class="setting-control">
<t-select
v-model="localProvider"
size="medium"
:placeholder="$t('kbSettings.storage.selectPlaceholder')"
style="width: 100%; min-width: 220px;"
@change="handleChange"
>
<t-option
v-for="opt in engineOptions"
:key="opt.value"
:value="opt.value"
:label="opt.label"
:disabled="opt.disabled"
>
<t-select v-model="localID" style="width:100%;min-width:260px" :disabled="!!props.hasFiles" @change="handleChange">
<t-option v-for="backend in backends" :key="backend.id" :value="backend.id" :label="backend.name">
<span class="select-option">
<span>{{ opt.label }}</span>
<t-tag v-if="opt.disabled && opt.allowed === false" theme="danger" variant="light" size="small">{{ $t('kbSettings.storage.unavailable') }}</t-tag>
<t-tag v-else-if="opt.disabled" theme="warning" variant="light" size="small">{{ $t('kbSettings.storage.notConfigured') }}</t-tag>
<t-tag v-else-if="opt.available === false" theme="danger" variant="light" size="small">{{ $t('kbSettings.storage.unavailable') }}</t-tag>
<span>{{ backend.name }}</span>
<t-tag theme="primary" variant="light" size="small">{{ backend.provider.toUpperCase() }}</t-tag>
<t-tag v-if="backend.id === defaultID" variant="light" size="small">默认</t-tag>
</span>
</t-option>
</t-select>
<p v-if="props.hasFiles" class="option-hint change-warning">{{ $t('kbSettings.storage.changeWarning') }}</p>
<p v-else-if="selectedOption?.desc" class="option-hint">{{ selectedOption.desc }}</p>
<a v-if="showGoSettings" href="javascript:void(0)" class="go-settings" @click.prevent="goToStorageSettings">{{ $t('kbSettings.storage.goGlobalSettings') }}</a>
<p v-if="props.hasFiles" class="option-hint change-warning">知识库中已有文件,需通过存储迁移流程才能更换实例。</p>
<p v-else-if="selected" class="option-hint">{{ selected.config.endpoint || selected.config.bucket_name || selected.config.path_prefix || '本地存储' }}</p>
<a href="javascript:void(0)" class="go-settings" @click.prevent="goToSettings">管理存储实例</a>
</div>
</div>
</div>
@@ -51,274 +31,39 @@
</template>
<script setup lang="ts">
import { ref, computed, watch, onMounted } from 'vue'
import { useI18n } from 'vue-i18n'
import { type StorageEngineStatusItem } from '@/api/system'
import { computed, onMounted, ref, watch } from 'vue'
import { listStorageBackends, type StorageBackend } from '@/api/storage-backend'
import { useUIStore } from '@/stores/ui'
import { useEditorResourcesStore } from '@/stores/editorResources'
const { t } = useI18n()
const props = defineProps<{
storageProvider: string
hasFiles?: boolean
}>()
const props = defineProps<{ storageBackendId?: string; storageProvider?: string; hasFiles?: boolean }>()
const emit = defineEmits<{
'update:storageBackendId': [value: string]
'update:storageProvider': [value: string]
}>()
const uiStore = useUIStore()
const editorResources = useEditorResourcesStore()
// Keep empty until tenant default_provider is loaded — do not pre-fill 'local'.
const localProvider = ref(props.storageProvider)
const loading = ref(true)
const engineStatus = ref<StorageEngineStatusItem[]>([])
const defaultProvider = ref('local')
const allowedProviders = ref<string[]>([])
const hasAnyConfig = ref(false)
const engineOptions = computed(() => {
const statusByName = new Map(engineStatus.value.map((engine) => [engine.name, engine]))
const allowedSet = new Set(allowedProviders.value)
const hasStatus = engineStatus.value.length > 0
const hasAllowedList = allowedProviders.value.length > 0
const resolveState = (name: string, fallbackAvailable = false) => {
const status = statusByName.get(name)
const allowed = status?.allowed ?? (hasAllowedList ? allowedSet.has(name) : !hasStatus)
const available = status?.available ?? fallbackAvailable
return {
allowed,
available,
disabled: !allowed || !available,
}
}
const local = resolveState('local', true)
const minio = resolveState('minio')
const cos = resolveState('cos')
const tos = resolveState('tos')
const s3 = resolveState('s3')
const oss = resolveState('oss')
const ks3 = resolveState('ks3')
const obs = resolveState('obs')
return [
{
value: 'local',
label: t('kbSettings.storage.engineLocal'),
desc: t('kbSettings.storage.engineLocalDesc'),
...local,
},
{
value: 'minio',
label: 'MinIO',
desc: t('kbSettings.storage.engineMinioDesc'),
...minio,
},
{
value: 'cos',
label: t('kbSettings.storage.engineCos'),
desc: t('kbSettings.storage.engineCosDesc'),
...cos,
},
{
value: 'tos',
label: t('kbSettings.storage.engineTos'),
desc: t('kbSettings.storage.engineTosDesc'),
...tos,
},
{
value: 's3',
label: t('kbSettings.storage.engineS3'),
desc: t('kbSettings.storage.engineS3Desc'),
...s3,
},
{
value: 'oss',
label: t('kbSettings.storage.engineOss'),
desc: t('kbSettings.storage.engineOssDesc'),
...oss,
},
{
value: 'ks3',
label: t('kbSettings.storage.engineKs3'),
desc: t('kbSettings.storage.engineKs3Desc'),
...ks3,
},
{
value: 'obs',
label: t('kbSettings.storage.engineObs'),
desc: t('kbSettings.storage.engineObsDesc'),
...obs,
},
]
})
const showGoSettings = computed(() =>
engineOptions.value.some(o => o.disabled)
)
const selectedOption = computed(() =>
engineOptions.value.find(o => o.value === localProvider.value)
)
const loading = ref(false), backends = ref<StorageBackend[]>([]), defaultID = ref(''), localID = ref(props.storageBackendId || '')
const selected = computed(() => backends.value.find(item => item.id === localID.value))
function handleChange() {
emit('update:storageProvider', localProvider.value)
emit('update:storageBackendId', localID.value)
emit('update:storageProvider', selected.value?.provider || props.storageProvider || '')
}
function ensureAllowedProvider() {
const resolved = editorResources.resolveUsableStorageProvider(localProvider.value)
if (resolved === localProvider.value) return
localProvider.value = resolved
emit('update:storageProvider', localProvider.value)
}
function goToStorageSettings() {
uiStore.closeKBEditor?.()
uiStore.openSettings?.('storage')
}
async function load(force = false) {
async function load() {
loading.value = true
try {
await editorResources.ensureStorageEngine(force)
const engines = editorResources.storageStatus
engineStatus.value = engines
allowedProviders.value = editorResources.storageAllowedProviders
defaultProvider.value = editorResources.resolveUsableStorageProvider(
editorResources.storageConfig?.default_provider,
)
const d = editorResources.storageConfig
hasAnyConfig.value = !!(d?.local?.path_prefix || d?.minio?.bucket_name || d?.cos?.bucket_name || d?.tos?.bucket_name || d?.s3?.bucket_name || d?.oss?.bucket_name || d?.ks3?.bucket_name || d?.obs?.bucket_name)
const parentUnset = !props.storageProvider
if (parentUnset) {
localProvider.value = defaultProvider.value
emit('update:storageProvider', localProvider.value)
} else {
localProvider.value = props.storageProvider
}
ensureAllowedProvider()
} catch {
engineStatus.value = []
} finally {
loading.value = false
}
const response = await listStorageBackends()
backends.value = (response.data || []).filter(item => item.status === 'active')
defaultID.value = response.default_storage_backend_id || ''
if (!localID.value) localID.value = defaultID.value || backends.value[0]?.id || ''
if (localID.value) handleChange()
} finally { loading.value = false }
}
// Sync only when parent sets an explicit provider (edit mode). Create mode leaves
// storageProvider empty until load() applies tenant default_provider.
watch(() => props.storageProvider, (v) => {
if (v) {
localProvider.value = v
}
})
function goToSettings() { uiStore.closeKBEditor?.(); uiStore.openSettings?.('storage') }
watch(() => props.storageBackendId, value => { if (value) localID.value = value })
onMounted(load)
</script>
<style lang="less" scoped>
.kb-storage-settings {
width: 100%;
}
.section-header {
margin-bottom: 20px;
h2 {
font-size: 20px;
font-weight: 600;
color: var(--td-text-color-primary);
margin: 0 0 6px 0;
}
.section-description {
font-size: 14px;
color: var(--td-text-color-secondary);
margin: 0;
line-height: 1.5;
}
}
.loading-inline {
display: flex;
align-items: center;
gap: 8px;
padding: 16px 0;
}
.settings-group {
display: flex;
flex-direction: column;
}
.setting-row {
display: flex;
align-items: flex-start;
justify-content: space-between;
padding: 16px 0;
border-bottom: 1px solid var(--td-component-stroke);
}
.setting-info {
flex: 0 0 40%;
max-width: 40%;
padding-right: 24px;
label {
font-size: 15px;
font-weight: 500;
color: var(--td-text-color-primary);
display: block;
margin-bottom: 4px;
}
.desc {
font-size: 13px;
color: var(--td-text-color-secondary);
margin: 0;
line-height: 1.5;
}
}
.setting-control {
flex: 0 0 55%;
max-width: 55%;
display: flex;
flex-direction: column;
align-items: flex-start;
gap: 6px;
}
.select-option {
display: inline-flex;
align-items: center;
gap: 8px;
}
.option-hint {
font-size: 12px;
color: var(--td-text-color-placeholder);
margin: 0;
line-height: 1.4;
&.locked-hint {
color: var(--td-warning-color);
}
&.change-warning {
color: var(--td-warning-color);
}
}
.go-settings {
font-size: 13px;
color: var(--td-brand-color, #0052d9);
margin-top: 8px;
text-decoration: none;
&:hover {
text-decoration: underline;
}
}
<style scoped lang="less">
.section-header{margin-bottom:20px}.section-header h2{font-size:20px;margin:0 0 6px}.section-description,.desc,.option-hint{color:var(--td-text-color-secondary)}
.setting-row{display:flex;justify-content:space-between;gap:28px}.setting-info{flex:1}.setting-control{width:45%;min-width:300px}.select-option{display:flex;align-items:center;gap:8px}.option-hint{font-size:12px;margin:8px 0}.change-warning{color:var(--td-warning-color)}.go-settings{font-size:13px;color:var(--td-brand-color)}.loading-inline{display:flex;gap:8px}
</style>
+1 -1
View File
@@ -196,7 +196,7 @@ import WebSearchSettings from './WebSearchSettings.vue'
import ChatHistorySettings from './ChatHistorySettings.vue'
import VectorStoreSettings from './VectorStoreSettings.vue'
import ParserEngineSettings from './ParserEngineSettings.vue'
import StorageEngineSettings from './StorageEngineSettings.vue'
import StorageEngineSettings from './StorageBackendSettings.vue'
import WeKnoraCloudSettings from './WeKnoraCloudSettings.vue'
import TenantMembers from './TenantMembers.vue'
import SystemSettings from '@/views/system/SystemSettings.vue'
@@ -0,0 +1,772 @@
<template>
<div class="storage-backend-settings">
<div class="section-header">
<div class="section-header__top">
<div>
<h2>{{ t('settings.storage.title') }}</h2>
<p class="section-description">管理文件与图片使用的存储实例;同一种类型可以配置多个实例。</p>
</div>
</div>
</div>
<t-loading :loading="loading" size="small" class="backend-list-loading">
<t-empty
v-if="!loading && backends.length === 0 && !authStore.hasRole('admin')"
description="尚未配置存储实例"
/>
<div v-else-if="!loading" class="backend-grid">
<div
v-for="backend in backends"
:key="backend.id"
class="backend-card"
:class="[
`backend-card--${backend.provider}`,
{ 'backend-card--clickable': canEdit(backend) },
]"
:role="canEdit(backend) ? 'button' : undefined"
:tabindex="canEdit(backend) ? 0 : undefined"
@click="onCardClick($event, backend)"
@keydown.enter="onCardClick($event, backend)"
>
<div
class="backend-card__badge"
:class="badgeClass(backend.provider)"
:style="badgeStyle(backend.provider)"
:aria-label="backend.provider"
>
<img
v-if="resolveLogo(backend.provider)?.mode === 'color'"
:src="resolveLogo(backend.provider)!.url"
:alt="backend.provider"
class="backend-card__badge-img"
/>
<template v-else-if="!resolveLogo(backend.provider)">{{ providerInitial(backend.provider) }}</template>
</div>
<div class="backend-card__body">
<div class="backend-card__header">
<h3 class="backend-card__title">{{ backend.name }}</h3>
<t-tag v-if="backend.id === defaultID" theme="primary" variant="light" size="small">默认</t-tag>
<div v-if="hasActions(backend)" class="backend-card__actions" @click.stop>
<t-dropdown
:options="getBackendOptions(backend)"
placement="bottom-right"
attach="body"
trigger="click"
@click="(data: any) => handleMenuAction(data.value, backend)"
>
<t-button variant="text" shape="square" size="small" class="backend-card__action-btn">
<t-icon name="ellipsis" />
</t-button>
</t-dropdown>
</div>
</div>
<p class="backend-card__subtitle">
<span>{{ backend.provider.toUpperCase() }}</span>
<template v-if="backendMeta(backend)">
<span class="backend-card__sep">·</span>
<span class="backend-card__meta">{{ backendMeta(backend) }}</span>
</template>
</p>
</div>
</div>
<button
v-if="authStore.hasRole('admin')"
type="button"
class="backend-card backend-card--add"
@click="openCreate"
>
<span class="backend-card--add__icon" aria-hidden="true">
<add-icon />
</span>
<span class="backend-card--add__label">添加存储实例</span>
</button>
</div>
</t-loading>
<SettingDrawer
v-model:visible="visible"
:title="editing ? '编辑存储实例' : '添加存储实例'"
:class="`storage-backend-drawer storage-backend-drawer--${form.provider}`"
:confirm-loading="saving"
@confirm="save"
@cancel="visible = false"
>
<template #headerIcon>
<img
v-if="currentLogo?.mode === 'color'"
:src="currentLogo.url"
:alt="form.provider"
class="header-icon__img"
/>
<span
v-else-if="currentLogo?.mode === 'mono'"
class="header-icon__mono"
:style="monoLogoStyle"
/>
<span v-else class="header-icon__text">{{ providerInitial(form.provider) }}</span>
</template>
<template #subtitle>
<span>{{ editing ? '修改此存储实例的连接配置。' : '为文件与图片新增一个存储实例。' }}</span>
</template>
<t-form :data="form" layout="vertical">
<section class="setting-drawer__section">
<h4 class="setting-drawer__section-title">基本信息</h4>
<div class="form-item">
<label class="form-label required">名称</label>
<t-input v-model="form.name" placeholder="例如:生产 COS、归档 COS" clearable />
</div>
<div class="form-item">
<label class="form-label required">存储类型</label>
<t-select v-model="form.provider" :disabled="!!editing" @change="resetConfig">
<t-option
v-for="provider in providers"
:key="provider"
:value="provider"
:label="provider.toUpperCase()"
/>
</t-select>
</div>
<div v-if="form.provider === 'minio'" class="form-item">
<label class="form-label">部署模式</label>
<div class="source-options" role="radiogroup">
<button
type="button"
class="source-option"
:class="{ 'is-active': form.config.mode !== 'docker' }"
:disabled="!!editing"
@click="form.config.mode = 'remote'"
>
<t-icon name="cloud" class="source-option__icon" />
<span class="source-option__label">远程实例</span>
</button>
<button
type="button"
class="source-option"
:class="{ 'is-active': form.config.mode === 'docker' }"
:disabled="!!editing"
@click="form.config.mode = 'docker'"
>
<t-icon name="server" class="source-option__icon" />
<span class="source-option__label">环境变量</span>
</button>
</div>
</div>
</section>
<section class="setting-drawer__section">
<h4 class="setting-drawer__section-title">连接配置</h4>
<div v-if="needsEndpoint" class="form-item">
<label class="form-label required">Endpoint</label>
<t-input
v-model="form.config.endpoint"
:disabled="!!editing"
:placeholder="form.provider === 'minio' ? 'storage.example.com:9000' : 'https://storage.example.com'"
clearable
/>
</div>
<div v-if="needsRegion" class="form-item">
<label class="form-label required">Region</label>
<t-input v-model="form.config.region" :disabled="!!editing" clearable />
</div>
<template v-if="needsCredentials">
<div class="form-item">
<label class="form-label required">Access Key / Secret ID</label>
<t-input v-model="form.config.access_key_id" placeholder="***" clearable>
<template #prefix-icon><t-icon name="lock-on" /></template>
</t-input>
</div>
<div class="form-item">
<label class="form-label required">Secret Key</label>
<t-input v-model="form.config.secret_access_key" type="password" placeholder="***" clearable>
<template #prefix-icon><t-icon name="lock-on" /></template>
</t-input>
</div>
</template>
<div v-if="form.provider !== 'local'" class="form-item">
<label class="form-label required">Bucket</label>
<t-input v-model="form.config.bucket_name" :disabled="!!editing" clearable />
</div>
<div v-if="form.provider === 'cos'" class="form-item">
<label class="form-label">App ID</label>
<t-input v-model="form.config.app_id" :disabled="!!editing" placeholder="可选" clearable />
</div>
</section>
<section class="setting-drawer__section">
<h4 class="setting-drawer__section-title">高级选项</h4>
<div class="form-item">
<label class="form-label">路径前缀</label>
<t-input v-model="form.config.path_prefix" :disabled="!!editing" placeholder="weknora/" clearable />
</div>
<div v-if="form.provider === 'minio'" class="form-item">
<div class="vision-toggle">
<t-switch v-model="form.config.use_ssl" />
<span class="form-desc form-desc--inline">使用 HTTPS 访问 MinIO</span>
</div>
</div>
<div v-if="form.provider === 's3'" class="form-item">
<div class="vision-toggle">
<t-switch v-model="form.config.force_path_style" />
<span class="form-desc form-desc--inline">使用 Path Style</span>
</div>
</div>
<div v-if="form.provider === 'oss'" class="form-item">
<div class="vision-toggle">
<t-switch v-model="form.config.use_temp_bucket" />
<span class="form-desc form-desc--inline">使用临时桶</span>
</div>
</div>
<template v-if="['cos', 'tos'].includes(form.provider) || (form.provider === 'oss' && form.config.use_temp_bucket)">
<div class="form-item">
<label class="form-label">临时桶</label>
<t-input v-model="form.config.temp_bucket_name" placeholder="可选,用于临时文件" clearable />
</div>
<div class="form-item">
<label class="form-label">临时桶 Region</label>
<t-input v-model="form.config.temp_region" placeholder="留空时使用主 Region" clearable />
</div>
</template>
</section>
</t-form>
<template #footer-left>
<t-button variant="outline" :loading="testing" @click="testRaw">
<template #icon>
<t-icon v-if="!testing && rawTestResult === 'ok'" name="check-circle-filled" class="status-icon available" />
<t-icon v-else-if="!testing && rawTestResult === 'error'" name="close-circle-filled" class="status-icon unavailable" />
</template>
测试连接
</t-button>
</template>
</SettingDrawer>
</div>
</template>
<script setup lang="ts">
import { computed, onMounted, reactive, ref } from 'vue'
import { DialogPlugin, MessagePlugin } from 'tdesign-vue-next'
import { AddIcon } from 'tdesign-icons-vue-next'
import { useI18n } from 'vue-i18n'
import { useAuthStore } from '@/stores/auth'
import SettingDrawer from '@/components/settings/SettingDrawer.vue'
import { providerLogo } from './providerLogos'
import {
createStorageBackend, deleteStorageBackend, listStorageBackends, listStorageBackendTypes,
setDefaultStorageBackend, testStorageBackend, testStorageBackendByID, updateStorageBackend,
type StorageBackend, type StorageBackendConfig,
} from '@/api/storage-backend'
const { t } = useI18n()
const authStore = useAuthStore()
const loading = ref(false), saving = ref(false), testing = ref(false), visible = ref(false)
const backends = ref<StorageBackend[]>([]), providers = ref<string[]>([]), defaultID = ref('')
const editing = ref<StorageBackend | null>(null)
const rawTestResult = ref<'ok' | 'error' | null>(null)
const blankConfig = (): StorageBackendConfig => ({ mode: 'remote', endpoint: '', region: '', access_key_id: '', secret_access_key: '', bucket_name: '', path_prefix: '', use_ssl: true })
const form = reactive<{ name: string; provider: string; config: StorageBackendConfig }>({ name: '', provider: 'local', config: blankConfig() })
const needsEndpoint = computed(() => !['local', 'cos'].includes(form.provider) && !(form.provider === 'minio' && form.config.mode === 'docker'))
const needsRegion = computed(() => !['local', 'minio'].includes(form.provider))
const needsCredentials = computed(() => form.provider !== 'local' && !(form.provider === 'minio' && form.config.mode === 'docker'))
const resolveLogo = (provider: string) => providerLogo('storage', provider)
const providerInitial = (provider: string) => (provider || '?').trim().charAt(0).toUpperCase() || '?'
const badgeClass = (provider: string) => {
const mode = resolveLogo(provider)?.mode
return {
'backend-card__badge--logo': !!mode,
'backend-card__badge--color': mode === 'color',
'backend-card__badge--mono': mode === 'mono',
}
}
const badgeStyle = (provider: string): Record<string, string> => {
const logo = resolveLogo(provider)
return logo?.mode === 'mono' ? { '--logo-url': `url("${logo.url}")` } : {}
}
const currentLogo = computed(() => providerLogo('storage', form.provider))
const monoLogoStyle = computed((): Record<string, string> => {
const logo = currentLogo.value
if (!logo || logo.mode !== 'mono') return {}
return { '--logo-url': `url("${logo.url}")` }
})
function backendMeta(backend: StorageBackend): string {
return backend.config.endpoint || backend.config.bucket_name || backend.config.path_prefix || '本地存储'
}
const canEdit = (backend: StorageBackend) => authStore.hasRole('admin') && backend.source !== 'env'
const canDelete = (backend: StorageBackend) => authStore.hasRole('admin') && backend.source !== 'env' && !backend.legacy_alias
const canSetDefault = (backend: StorageBackend) => backend.id !== defaultID.value && authStore.hasRole('admin')
// 测试连接对所有可见用户开放,因此每张卡至少有一个动作。
const hasActions = (_backend: StorageBackend) => true
function getBackendOptions(backend: StorageBackend) {
const options: { content: string; value: string; theme?: string }[] = []
options.push({ content: '测试连接', value: 'test' })
if (canSetDefault(backend)) options.push({ content: '设为默认', value: 'default' })
if (canEdit(backend)) options.push({ content: '编辑', value: 'edit' })
if (canDelete(backend)) options.push({ content: '删除', value: 'delete', theme: 'error' })
return options
}
function handleMenuAction(value: string, backend: StorageBackend) {
if (value === 'test') testSaved(backend)
else if (value === 'default') makeDefault(backend)
else if (value === 'edit') openEdit(backend)
else if (value === 'delete') remove(backend)
}
function onCardClick(event: Event, backend: StorageBackend) {
if (!canEdit(backend)) return
const target = event.target as HTMLElement | null
if (target?.closest('.backend-card__actions')) return
openEdit(backend)
}
async function load() {
loading.value = true
try {
const [list, types] = await Promise.all([listStorageBackends(), listStorageBackendTypes()])
backends.value = list.data || []; defaultID.value = list.default_storage_backend_id || ''; providers.value = types.data || []
} finally { loading.value = false }
}
function resetConfig() { form.config = blankConfig(); rawTestResult.value = null }
function openCreate() { editing.value = null; form.name = ''; form.provider = providers.value[0] || 'local'; form.config = blankConfig(); rawTestResult.value = null; visible.value = true }
function openEdit(backend: StorageBackend) { editing.value = backend; form.name = backend.name; form.provider = backend.provider; form.config = { ...blankConfig(), ...backend.config }; rawTestResult.value = null; visible.value = true }
async function testRaw() {
testing.value = true
rawTestResult.value = null
try {
const r: any = editing.value ? await testStorageBackendByID(editing.value.id) : await testStorageBackend(form)
if (r.success) { rawTestResult.value = 'ok'; MessagePlugin.success('连接成功') }
else { rawTestResult.value = 'error'; MessagePlugin.error(r.error || '连接失败') }
} finally { testing.value = false }
}
async function testSaved(backend: StorageBackend) { const r: any = await testStorageBackendByID(backend.id); r.success ? MessagePlugin.success('连接成功') : MessagePlugin.error(r.error || '连接失败') }
async function save() {
if (!form.name.trim()) { MessagePlugin.warning('请输入名称'); return }
saving.value = true
try {
const payload = { name: form.name.trim(), provider: form.provider, config: { ...form.config } }
if (editing.value) await updateStorageBackend(editing.value.id, payload); else await createStorageBackend(payload)
MessagePlugin.success('保存成功'); visible.value = false; await load()
} catch (e: any) { MessagePlugin.error(e?.message || '保存失败') } finally { saving.value = false }
}
async function makeDefault(backend: StorageBackend) { await setDefaultStorageBackend(backend.id); defaultID.value = backend.id; MessagePlugin.success('默认存储已更新') }
function remove(backend: StorageBackend) {
const dialog = DialogPlugin.confirm({ header: '删除存储实例', body: `确定删除“${backend.name}”吗?`, onConfirm: async () => { dialog.destroy(); try { await deleteStorageBackend(backend.id); await load(); MessagePlugin.success('已删除') } catch (e: any) { MessagePlugin.error(e?.message || '删除失败') } }, onCancel: () => dialog.destroy() })
}
onMounted(load)
</script>
<style scoped lang="less">
.storage-backend-settings {
width: 100%;
}
.section-header {
margin-bottom: 28px;
h2 {
font-size: 20px;
font-weight: 600;
color: var(--td-text-color-primary);
margin: 0 0 8px 0;
}
.section-description {
font-size: 14px;
color: var(--td-text-color-secondary);
margin: 0;
line-height: 1.6;
}
}
.section-header__top {
display: flex;
align-items: center;
justify-content: space-between;
gap: 20px;
}
.backend-list-loading {
min-height: 120px;
}
.backend-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(320px, 1fr));
gap: 12px;
.backend-card--add {
width: 100%;
height: 100%;
}
}
.backend-card {
position: relative;
display: flex;
align-items: flex-start;
gap: 12px;
padding: 14px 16px;
border: 1px solid var(--td-component-stroke);
border-radius: 10px;
background: var(--td-bg-color-container);
transition: border-color 0.18s ease, box-shadow 0.18s ease;
min-width: 0;
&:hover {
border-color: var(--td-brand-color-3, var(--td-brand-color));
box-shadow: 0 4px 14px rgba(15, 23, 42, 0.06);
}
&--clickable {
cursor: pointer;
&:focus-visible {
outline: 2px solid var(--td-brand-color);
outline-offset: 2px;
}
}
&--add {
flex-direction: column;
align-items: center;
justify-content: center;
gap: 8px;
min-height: 68px;
border-style: dashed;
background: transparent;
color: var(--td-text-color-placeholder);
cursor: pointer;
font: inherit;
text-align: center;
&:hover,
&:focus-visible {
color: var(--td-brand-color);
border-color: var(--td-brand-color);
background: color-mix(in srgb, var(--td-brand-color) 6%, transparent);
box-shadow: none;
}
&:focus-visible {
outline: 2px solid var(--td-brand-color);
outline-offset: 2px;
}
&__icon {
display: flex;
align-items: center;
justify-content: center;
width: 32px;
height: 32px;
border-radius: 8px;
background: color-mix(in srgb, var(--td-brand-color) 10%, transparent);
color: var(--td-brand-color);
font-size: 18px;
}
&__label {
font-size: 13px;
font-weight: 500;
line-height: 1.4;
}
}
}
.backend-card__badge {
flex-shrink: 0;
width: 36px;
height: 36px;
border-radius: 9px;
display: flex;
align-items: center;
justify-content: center;
margin-top: 1px;
font-size: 15px;
font-weight: 600;
letter-spacing: 0.02em;
background: rgba(0, 82, 217, 0.1);
color: #0052D9;
}
.backend-card .backend-card__badge--logo {
background: var(--td-bg-color-container, #fff);
box-shadow: inset 0 0 0 1px var(--td-component-stroke);
}
.backend-card .backend-card__badge--mono::before {
content: '';
width: 22px;
height: 22px;
background-color: currentColor;
-webkit-mask-image: var(--logo-url);
-webkit-mask-position: center;
-webkit-mask-repeat: no-repeat;
-webkit-mask-size: contain;
mask-image: var(--logo-url);
mask-position: center;
mask-repeat: no-repeat;
mask-size: contain;
}
.backend-card__badge-img {
width: 24px;
height: 24px;
object-fit: contain;
display: block;
}
.backend-card--local .backend-card__badge { background: rgba(70, 70, 70, 0.1); color: #464646; }
.backend-card--minio .backend-card__badge { background: rgba(225, 38, 38, 0.12); color: #C0382B; }
.backend-card--cos .backend-card__badge { background: rgba(0, 82, 217, 0.1); color: #0052D9; }
.backend-card--tos .backend-card__badge { background: rgba(0, 137, 255, 0.12); color: #0089FF; }
.backend-card--s3 .backend-card__badge { background: rgba(255, 153, 0, 0.12); color: #D97706; }
.backend-card--oss .backend-card__badge { background: rgba(255, 90, 0, 0.12); color: #E55A00; }
.backend-card--ks3 .backend-card__badge { background: rgba(7, 192, 95, 0.12); color: #07A050; }
.backend-card--obs .backend-card__badge { background: rgba(206, 17, 38, 0.1); color: #CE1126; }
.backend-card__body {
flex: 1;
min-width: 0;
display: flex;
flex-direction: column;
justify-content: center;
gap: 2px;
}
.backend-card__header {
display: flex;
align-items: center;
gap: 6px;
min-width: 0;
}
.backend-card__title {
flex: 1;
min-width: 0;
margin: 0;
font-size: 14px;
font-weight: 600;
line-height: 1.4;
color: var(--td-text-color-primary);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.backend-card__subtitle {
margin: 2px 0 0;
font-size: 12px;
line-height: 1.5;
color: var(--td-text-color-secondary);
display: flex;
align-items: center;
min-width: 0;
}
.backend-card__sep {
margin: 0 6px;
color: var(--td-text-color-placeholder);
flex-shrink: 0;
}
.backend-card__meta {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
min-width: 0;
}
.backend-card__actions {
flex-shrink: 0;
display: flex;
align-items: center;
}
.backend-card__action-btn {
flex-shrink: 0;
padding: 2px;
color: var(--td-text-color-placeholder);
opacity: 0;
transition: opacity 0.15s ease;
&:hover,
&:focus-visible {
background: var(--td-bg-color-secondarycontainer);
color: var(--td-text-color-primary);
}
}
.backend-card:hover .backend-card__action-btn,
.backend-card:focus-within .backend-card__action-btn {
opacity: 1;
}
// ---- 抽屉头部图标 ----
.header-icon__img {
width: 24px;
height: 24px;
object-fit: contain;
display: block;
}
.header-icon__mono {
display: inline-block;
width: 22px;
height: 22px;
background-color: currentColor;
-webkit-mask-image: var(--logo-url);
-webkit-mask-position: center;
-webkit-mask-repeat: no-repeat;
-webkit-mask-size: contain;
mask-image: var(--logo-url);
mask-position: center;
mask-repeat: no-repeat;
mask-size: contain;
}
.header-icon__text {
font-size: 15px;
font-weight: 600;
letter-spacing: 0.02em;
}
// ---- 抽屉表单 ----
.form-item {
margin-bottom: 0;
}
.form-label {
display: block;
margin-bottom: 6px;
font-size: 13px;
font-weight: 500;
color: var(--td-text-color-primary);
line-height: 1.4;
&.required::before {
content: '*';
color: var(--td-error-color);
margin-right: 4px;
font-weight: 500;
line-height: 1;
}
}
.form-desc {
margin: 4px 0 0 0;
font-size: 12px;
line-height: 1.5;
color: var(--td-text-color-placeholder);
&--inline {
margin: 0;
}
}
:deep(.t-input),
:deep(.t-select),
:deep(.t-textarea) {
width: 100%;
font-size: 13px;
}
.vision-toggle {
display: flex;
align-items: center;
gap: 8px;
}
// ---- MinIO 部署模式 pill segmented ----
.source-options {
display: inline-flex;
align-items: center;
gap: 4px;
padding: 3px;
background: var(--td-bg-color-component);
border: 1px solid var(--td-component-stroke);
border-radius: 8px;
}
.source-option {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 5px 12px;
height: 28px;
background: transparent;
border: 1px solid transparent;
border-radius: 6px;
cursor: pointer;
font-family: inherit;
font-size: 13px;
color: var(--td-text-color-secondary);
line-height: 1;
transition: all 0.15s ease;
&:disabled {
cursor: not-allowed;
opacity: 0.6;
}
&:hover:not(.is-active):not(:disabled) {
color: var(--td-text-color-primary);
background: var(--td-bg-color-container-hover);
}
&.is-active {
background: var(--td-bg-color-container);
border-color: var(--td-brand-color);
color: var(--td-brand-color);
font-weight: 500;
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.04);
}
}
.source-option__icon {
font-size: 14px;
flex-shrink: 0;
}
.source-option__label {
white-space: nowrap;
}
.status-icon {
font-size: 16px;
flex-shrink: 0;
&.available {
color: var(--td-brand-color);
}
&.unavailable {
color: var(--td-error-color);
}
}
</style>
<!--
Non-scoped: drawer header icon coloring per provider, mirroring the list
card badge colors. Namespaced under .storage-backend-drawer--{id}.
-->
<style lang="less">
.storage-backend-drawer .setting-drawer__header-icon:has(.header-icon__img) {
background: var(--td-bg-color-container, #fff);
box-shadow: inset 0 0 0 1px var(--td-component-stroke);
}
.storage-backend-drawer--local .setting-drawer__header-icon { background: rgba(70, 70, 70, 0.1); color: #464646; }
.storage-backend-drawer--minio .setting-drawer__header-icon { background: rgba(225, 38, 38, 0.12); color: #C0382B; }
.storage-backend-drawer--cos .setting-drawer__header-icon { background: rgba(0, 82, 217, 0.1); color: #0052D9; }
.storage-backend-drawer--tos .setting-drawer__header-icon { background: rgba(0, 137, 255, 0.12); color: #0089FF; }
.storage-backend-drawer--s3 .setting-drawer__header-icon { background: rgba(255, 153, 0, 0.12); color: #D97706; }
.storage-backend-drawer--oss .setting-drawer__header-icon { background: rgba(255, 90, 0, 0.12); color: #E55A00; }
.storage-backend-drawer--ks3 .setting-drawer__header-icon { background: rgba(7, 192, 95, 0.12); color: #07A050; }
.storage-backend-drawer--obs .setting-drawer__header-icon { background: rgba(206, 17, 38, 0.1); color: #CE1126; }
</style>
@@ -113,7 +113,7 @@
- color logo(如 MinIO/AWS): 直接 <img>,保留品牌彩色
- mono logo(mask-image): 通过 ::before + currentColor 染色,颜色由
.storage-engine-drawer--{id} :deep(.setting-drawer__header-icon) 决定
- 无 logo(local): 渲染首字母作为 monogram
- 无 logo: 渲染首字母作为 monogram
-->
<template v-if="currentEngine" #headerIcon>
<img
+25 -3
View File
@@ -122,7 +122,8 @@ type DataAnalysisTool struct {
// env var at request time can produce a different (or empty) value if the
// variable was not exported to the sub-process or was set programmatically
// after startup, causing GetFile to look in the wrong directory (#1040).
localBaseDir string
localBaseDir string
storageResolver interfaces.StorageBackendResolver
}
func NewDataAnalysisTool(
@@ -132,8 +133,9 @@ func NewDataAnalysisTool(
fileService interfaces.FileService,
db *sql.DB,
sessionID string,
storageResolvers ...interfaces.StorageBackendResolver,
) *DataAnalysisTool {
return &DataAnalysisTool{
tool := &DataAnalysisTool{
BaseTool: dataAnalysisTool,
knowledgeBaseService: knowledgeBaseService,
knowledgeService: knowledgeService,
@@ -145,8 +147,12 @@ func NewDataAnalysisTool(
// call to resolveFileServiceForKnowledge uses the same base path. The
// env var is guaranteed to be set (or empty == "/data/files" fallback)
// when the application starts and the DI container is assembled.
localBaseDir: strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")),
localBaseDir: strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")),
}
if len(storageResolvers) > 0 {
tool.storageResolver = storageResolvers[0]
}
return tool
}
// recordCreatedTable records a table name for cleanup, ensuring uniqueness
@@ -782,8 +788,12 @@ func (t *DataAnalysisTool) resolveFileServiceForKnowledge(ctx context.Context, k
}
provider := ""
backendID, _, _ := types.ParseStorageBackendPath(knowledge.FilePath)
if kb != nil {
provider = kb.GetStorageProvider()
if backendID == "" && kb.StorageBackendID != nil {
backendID = strings.TrimSpace(*kb.StorageBackendID)
}
}
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
@@ -809,6 +819,18 @@ func (t *DataAnalysisTool) resolveFileServiceForKnowledge(ctx context.Context, k
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
}
if t.storageResolver != nil && tenant != nil && (backendID != "" || provider != "") {
resolvedSvc, resolvedProvider, err := t.storageResolver.ResolveFileService(
ctx, tenant, backendID, provider, t.localBaseDir,
)
if err == nil {
logger.Infof(ctx, "[Tool][DataAnalysis][storage] resolved storage backend: session_id=%s knowledge_id=%s kb_id=%s backend_id=%s provider=%s",
t.sessionID, knowledge.ID, kbID, backendID, resolvedProvider)
return resolvedSvc
}
logger.Warnf(ctx, "[Tool][DataAnalysis][storage] resolve storage backend failed, trying legacy config: session_id=%s knowledge_id=%s kb_id=%s backend_id=%s provider=%s err=%v",
t.sessionID, knowledge.ID, kbID, backendID, provider, err)
}
if provider == "" || tenant == nil || tenant.StorageEngineConfig == nil {
hasTenantStorageConfig := tenant != nil && tenant.StorageEngineConfig != nil
@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
pinned_at DATETIME NULL,
asr_config TEXT,
vector_store_id VARCHAR(36),
storage_backend_id VARCHAR(36),
wiki_config TEXT,
indexing_strategy TEXT,
creator_id VARCHAR(36),
@@ -0,0 +1,58 @@
package repository
import (
"context"
"errors"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
)
type storageBackendRepository struct{ db *gorm.DB }
func NewStorageBackendRepository(db *gorm.DB) interfaces.StorageBackendRepository {
return &storageBackendRepository{db: db}
}
func (r *storageBackendRepository) Create(ctx context.Context, backend *types.StorageBackend) error {
return r.db.WithContext(ctx).Create(backend).Error
}
func (r *storageBackendRepository) GetByID(ctx context.Context, tenantID uint64, id string) (*types.StorageBackend, error) {
var backend types.StorageBackend
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).First(&backend).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return &backend, nil
}
func (r *storageBackendRepository) List(ctx context.Context, tenantID uint64) ([]*types.StorageBackend, error) {
var backends []*types.StorageBackend
err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).Order("created_at DESC").Find(&backends).Error
return backends, err
}
func (r *storageBackendRepository) Update(ctx context.Context, backend *types.StorageBackend) error {
return r.db.WithContext(ctx).Model(&types.StorageBackend{}).
Where("tenant_id = ? AND id = ?", backend.TenantID, backend.ID).
Select("name", "config", "status", "updated_at").Updates(backend).Error
}
func (r *storageBackendRepository) Delete(ctx context.Context, tenantID uint64, id string) error {
return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.StorageBackend{}).Error
}
func (r *storageBackendRepository) FindLegacyAlias(ctx context.Context, tenantID uint64, provider string) (*types.StorageBackend, error) {
var backend types.StorageBackend
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND provider = ? AND legacy_alias = ?", tenantID, provider, true).First(&backend).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return &backend, nil
}
@@ -100,6 +100,7 @@ type agentService struct {
webSearchStateService interfaces.WebSearchStateService
wikiPageService interfaces.WikiPageService
tenantService interfaces.TenantService
storageResolver interfaces.StorageBackendResolver
toolApprovalGate approval.MCPApproval
}
@@ -120,6 +121,7 @@ func NewAgentService(
webSearchStateService interfaces.WebSearchStateService,
wikiPageService interfaces.WikiPageService,
tenantService interfaces.TenantService,
storageResolver interfaces.StorageBackendResolver,
toolApprovalGate approval.MCPApproval,
) interfaces.AgentService {
return &agentService{
@@ -138,6 +140,7 @@ func NewAgentService(
webSearchStateService: webSearchStateService,
wikiPageService: wikiPageService,
tenantService: tenantService,
storageResolver: storageResolver,
toolApprovalGate: toolApprovalGate,
}
}
@@ -622,7 +625,7 @@ func (s *agentService) registerTools(
logger.Infof(ctx, "Registered web_fetch tool for session: %s", sessionID)
case tools.ToolDataAnalysis:
toolToRegister = tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, s.fileService, s.duckdb, sessionID)
toolToRegister = tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, s.fileService, s.duckdb, sessionID, s.storageResolver)
logger.Infof(ctx, "Registered data_analysis tool for session: %s", sessionID)
case tools.ToolDataSchema:
+19 -12
View File
@@ -11,7 +11,6 @@ import (
"github.com/Tencent/WeKnora/internal/agent/tools"
chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
@@ -406,6 +405,7 @@ type DataTableSummaryService struct {
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
sqlDB *sql.DB
storageResolver interfaces.StorageBackendResolver
}
// NewDataTableSummaryService creates a new DataTableSummaryService
@@ -419,6 +419,7 @@ func NewDataTableSummaryService(
retrieveEngine interfaces.RetrieveEngineRegistry,
ownership retriever.TenantStoreOwnership,
sqlDB *sql.DB,
storageResolver interfaces.StorageBackendResolver,
) interfaces.TaskHandler {
return &DataTableSummaryService{
modelService: modelService,
@@ -430,6 +431,7 @@ func NewDataTableSummaryService(
retrieveEngine: retrieveEngine,
ownership: ownership,
sqlDB: sqlDB,
storageResolver: storageResolver,
}
}
@@ -559,24 +561,29 @@ func (s *DataTableSummaryService) resolveFileServiceForKnowledge(ctx context.Con
if resources == nil || resources.knowledge == nil {
return s.fileService
}
if resources.tenant == nil || resources.tenant.StorageEngineConfig == nil {
if resources.tenant == nil {
return s.fileService
}
provider := types.InferStorageFromFilePath(resources.knowledge.FilePath)
if provider == "" {
if provider == "" && resources.tenant.StorageEngineConfig != nil {
provider = strings.ToLower(strings.TrimSpace(resources.tenant.StorageEngineConfig.DefaultProvider))
}
if provider == "" {
return s.fileService
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
resolvedSvc, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig(
provider,
resources.tenant.StorageEngineConfig,
baseDir,
)
backendID, _, _ := types.ParseStorageBackendPath(resources.knowledge.FilePath)
if backendID == "" && resources.knowledgeBase != nil && resources.knowledgeBase.StorageBackendID != nil {
backendID = strings.TrimSpace(*resources.knowledgeBase.StorageBackendID)
}
// New-model workspaces resolve via DefaultStorageBackendID even when no
// legacy StorageEngineConfig / provider is present, so gate on the resolver
// and a usable backendID/provider rather than requiring a non-empty provider.
if s.storageResolver == nil || (backendID == "" && provider == "") {
return s.fileService
}
resolvedSvc, resolvedProvider, err := s.storageResolver.ResolveFileService(ctx, resources.tenant, backendID, provider, baseDir)
if err != nil {
logger.Warnf(ctx, "[TableSummary] Failed to resolve file service for provider=%s, fallback to default: %v", provider, err)
return s.fileService
@@ -591,7 +598,7 @@ func (s *DataTableSummaryService) processTableData(ctx context.Context, resource
// 创建DuckDB会话并加载数据
sessionID := fmt.Sprintf("table_summary_%s", resources.knowledge.ID)
fileSvc := s.resolveFileServiceForKnowledge(ctx, resources)
duckdbTool := tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, fileSvc, s.sqlDB, sessionID)
duckdbTool := tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, fileSvc, s.sqlDB, sessionID, s.storageResolver)
defer duckdbTool.Cleanup(ctx)
// 使用knowledge.ID作为表名,根据文件类型自动加载数据
@@ -0,0 +1,113 @@
package file
import (
"context"
"fmt"
"io"
"mime/multipart"
"net/url"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// backendScopedFileService makes the storage instance part of every newly
// persisted path while delegating actual I/O to the existing provider driver.
type backendScopedFileService struct {
backendID string
inner interfaces.FileService
}
func NewBackendScopedFileService(backendID string, inner interfaces.FileService) interfaces.FileService {
return &backendScopedFileService{backendID: backendID, inner: inner}
}
func (s *backendScopedFileService) unwrap(path string) (string, error) {
id, inner, ok := types.ParseStorageBackendPath(path)
if !ok {
return path, nil
}
if id != s.backendID {
return "", fmt.Errorf("storage backend mismatch")
}
return inner, nil
}
func (s *backendScopedFileService) wrap(path string) string {
return types.BuildStorageBackendPath(s.backendID, path)
}
func (s *backendScopedFileService) CheckConnectivity(ctx context.Context) error {
return s.inner.CheckConnectivity(ctx)
}
func (s *backendScopedFileService) SaveFile(ctx context.Context, f *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
p, err := s.inner.SaveFile(ctx, f, tenantID, knowledgeID)
if err != nil {
return "", err
}
return s.wrap(p), nil
}
func (s *backendScopedFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, name string, temp bool) (string, error) {
p, err := s.inner.SaveBytes(ctx, data, tenantID, name, temp)
if err != nil {
return "", err
}
return s.wrap(p), nil
}
func (s *backendScopedFileService) GetFile(ctx context.Context, path string) (io.ReadCloser, error) {
p, err := s.unwrap(path)
if err != nil {
return nil, err
}
return s.inner.GetFile(ctx, p)
}
func (s *backendScopedFileService) GetFileURL(ctx context.Context, path string) (string, error) {
p, err := s.unwrap(path)
if err != nil {
return "", err
}
result, err := s.inner.GetFileURL(ctx, p)
if err != nil {
return "", err
}
scoped := s.wrap(p)
if result == p {
return scoped, nil
}
// Local storage may return an app-level presigned URL. Re-sign it with
// the scoped path so the proxy resolves the exact local instance instead
// of falling back to another backend of the same provider.
if u, parseErr := url.Parse(result); parseErr == nil && strings.HasSuffix(u.Path, "/api/v1/files/presigned") && u.Query().Get("file_path") == p {
basePath := strings.TrimSuffix(u.Path, "/api/v1/files/presigned")
baseURL := u.Scheme + "://" + u.Host + basePath
ttl := time.Duration(0)
if expires, convErr := strconv.ParseInt(u.Query().Get("expires"), 10, 64); convErr == nil {
ttl = time.Until(time.Unix(expires, 0))
}
if signed, signErr := secutils.SignFileURL(baseURL, scoped, secutils.ParseTenantIDFromStoragePath(scoped), ttl); signErr == nil {
return signed, nil
}
}
return result, nil
}
func (s *backendScopedFileService) DeleteFile(ctx context.Context, path string) error {
p, err := s.unwrap(path)
if err != nil {
return err
}
return s.inner.DeleteFile(ctx, p)
}
func (s *backendScopedFileService) CopyFile(ctx context.Context, path string, tenantID uint64, knowledgeID string) (string, error) {
p, err := s.unwrap(path)
if err != nil {
return "", err
}
result, err := s.inner.CopyFile(ctx, p, tenantID, knowledgeID)
if err != nil {
return "", err
}
return s.wrap(result), nil
}
@@ -0,0 +1,34 @@
package file
import (
"context"
"io"
"net/url"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBackendScopedLocalURLRetainsBackendID(t *testing.T) {
t.Setenv("SYSTEM_AES_KEY", "0123456789abcdef0123456789abcdef")
inner := NewLocalFileService(t.TempDir(), "https://weknora.example.com/base")
svc := NewBackendScopedFileService("backend-local-a", inner)
path, err := svc.SaveBytes(context.Background(), []byte("hello"), 7, "exports/image.txt", false)
require.NoError(t, err)
assert.Contains(t, path, "storage://backend-local-a/local://")
signed, err := svc.GetFileURL(context.Background(), path)
require.NoError(t, err)
u, err := url.Parse(signed)
require.NoError(t, err)
assert.Equal(t, path, u.Query().Get("file_path"))
reader, err := svc.GetFile(context.Background(), path)
require.NoError(t, err)
defer reader.Close()
data, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "hello", string(data))
}
+31 -9
View File
@@ -82,14 +82,14 @@ func NewFileServiceFromStorageConfig(
if pathPrefix == "" {
pathPrefix = "weknora"
}
svc, err := NewCosFileService(sec.COS.BucketName, sec.COS.Region, sec.COS.SecretID, sec.COS.SecretKey, pathPrefix)
svc, err := NewCosFileServiceWithTempBucket(sec.COS.BucketName, sec.COS.Region, sec.COS.SecretID, sec.COS.SecretKey, pathPrefix, sec.COS.TempBucketName, sec.COS.TempRegion)
return svc, p, err
case "tos":
if sec == nil || sec.TOS == nil || sec.TOS.Endpoint == "" || sec.TOS.Region == "" || sec.TOS.AccessKey == "" || sec.TOS.SecretKey == "" || sec.TOS.BucketName == "" {
return nil, p, fmt.Errorf("incomplete tos config")
}
svc, err := NewTosFileService(sec.TOS.Endpoint, sec.TOS.Region, sec.TOS.AccessKey, sec.TOS.SecretKey, sec.TOS.BucketName, sec.TOS.PathPrefix)
svc, err := NewTosFileServiceWithTempBucket(sec.TOS.Endpoint, sec.TOS.Region, sec.TOS.AccessKey, sec.TOS.SecretKey, sec.TOS.BucketName, sec.TOS.PathPrefix, sec.TOS.TempBucketName, sec.TOS.TempRegion)
return svc, p, err
case "s3":
if sec == nil || sec.S3 == nil || sec.S3.Endpoint == "" || sec.S3.Region == "" || sec.S3.AccessKey == "" || sec.S3.SecretKey == "" || sec.S3.BucketName == "" {
@@ -99,16 +99,38 @@ func NewFileServiceFromStorageConfig(
if pathPrefix == "" {
pathPrefix = "weknora/"
}
svc, err := NewS3FileService(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix)
svc, err := NewS3FileServiceWithOptions(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix, sec.S3.ForcePathStyle)
return svc, p, err
case "obs":
obsEndpoint := strings.TrimSpace(os.Getenv("OBS_ENDPOINT"))
obsRegion := strings.TrimSpace(os.Getenv("OBS_REGION"))
obsAccessKey := strings.TrimSpace(os.Getenv("OBS_ACCESS_KEY"))
obsSecretKey := strings.TrimSpace(os.Getenv("OBS_SECRET_KEY"))
obsBucketName := strings.TrimSpace(os.Getenv("OBS_BUCKET_NAME"))
obsPathPrefix := strings.TrimSpace(os.Getenv("OBS_PATH_PREFIX"))
obsEndpoint, obsRegion, obsAccessKey := "", "", ""
obsSecretKey, obsBucketName, obsPathPrefix := "", "", ""
if sec != nil && sec.OBS != nil {
obsEndpoint = strings.TrimSpace(sec.OBS.Endpoint)
obsRegion = strings.TrimSpace(sec.OBS.Region)
obsAccessKey = strings.TrimSpace(sec.OBS.AccessKey)
obsSecretKey = strings.TrimSpace(sec.OBS.SecretKey)
obsBucketName = strings.TrimSpace(sec.OBS.BucketName)
obsPathPrefix = strings.TrimSpace(sec.OBS.PathPrefix)
}
if obsEndpoint == "" {
obsEndpoint = strings.TrimSpace(os.Getenv("OBS_ENDPOINT"))
}
if obsRegion == "" {
obsRegion = strings.TrimSpace(os.Getenv("OBS_REGION"))
}
if obsAccessKey == "" {
obsAccessKey = strings.TrimSpace(os.Getenv("OBS_ACCESS_KEY"))
}
if obsSecretKey == "" {
obsSecretKey = strings.TrimSpace(os.Getenv("OBS_SECRET_KEY"))
}
if obsBucketName == "" {
obsBucketName = strings.TrimSpace(os.Getenv("OBS_BUCKET_NAME"))
}
if obsPathPrefix == "" {
obsPathPrefix = strings.TrimSpace(os.Getenv("OBS_PATH_PREFIX"))
}
if obsPathPrefix == "" {
obsPathPrefix = "weknora/"
}
+27 -4
View File
@@ -30,7 +30,7 @@ type s3FileService struct {
}
// newS3Client creates a bare s3FileService with just the SDK client initialised.
func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string) (*s3FileService, error) {
func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (*s3FileService, error) {
var cfg aws.Config
var err error
@@ -49,7 +49,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
// (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
var client *s3.Client
if endpoint != "" {
usePathStyle := !strings.Contains(endpoint, "amazonaws.com")
usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(endpoint)
o.UsePathStyle = usePathStyle
@@ -76,7 +76,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
func NewS3FileService(endpoint,
accessKey, secretKey, bucketName, region, pathPrefix string,
) (interfaces.FileService, error) {
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix)
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix, false)
if err != nil {
return nil, err
}
@@ -96,6 +96,25 @@ func NewS3FileService(endpoint,
return svc, nil
}
// NewS3FileServiceWithOptions is the instance-aware S3 constructor. Existing
// callers keep the historical endpoint-based path-style inference.
func NewS3FileServiceWithOptions(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (interfaces.FileService, error) {
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix, forcePathStyle)
if err != nil {
return nil, err
}
exists, err := svc.bucketExists(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to check bucket: %w", err)
}
if !exists {
if err = svc.createBucket(context.Background()); err != nil {
return nil, fmt.Errorf("failed to create bucket: %w", err)
}
}
return svc, nil
}
// bucketExists checks if the bucket exists
func (s *s3FileService) bucketExists(ctx context.Context) (bool, error) {
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
@@ -145,7 +164,11 @@ func (s *s3FileService) CheckConnectivity(ctx context.Context) error {
// CheckS3Connectivity tests S3 connectivity using the provided credentials.
// It creates a temporary service instance internally and delegates to CheckConnectivity.
func CheckS3Connectivity(ctx context.Context, endpoint, accessKey, secretKey, bucketName, region string) error {
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, "")
return CheckS3ConnectivityWithOptions(ctx, endpoint, accessKey, secretKey, bucketName, region, false)
}
func CheckS3ConnectivityWithOptions(ctx context.Context, endpoint, accessKey, secretKey, bucketName, region string, forcePathStyle bool) error {
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, "", forcePathStyle)
if err != nil {
return err
}
@@ -11,7 +11,6 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/application/repository"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
@@ -79,7 +78,8 @@ type ImageMultimodalService struct {
// (e.g. images were saved using the global MINIO_* env vars while the
// tenant's StorageEngineConfig.MinIO is empty). Mirrors the write-side
// fallback in knowledgeService.resolveFileService.
fileSvc interfaces.FileService
fileSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
// spanTracker records this image's subspan under the parent attempt's
// multimodal stage. nil-safe — falls back to no-op via tracker().
@@ -98,21 +98,23 @@ func NewImageMultimodalService(
taskEnqueuer interfaces.TaskEnqueuer,
redisClient *redis.Client,
fileSvc interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
spanTracker SpanTracker,
) interfaces.TaskHandler {
return &ImageMultimodalService{
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ownership: ownership,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
redisClient: redisClient,
fileSvc: fileSvc,
spanTracker: spanTracker,
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ownership: ownership,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
redisClient: redisClient,
fileSvc: fileSvc,
storageResolver: storageResolver,
spanTracker: spanTracker,
}
}
@@ -550,6 +552,7 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
return s.fileSvc
}
backendID, _, _ := types.ParseStorageBackendPath(payload.ImageURL)
provider := types.ParseProviderScheme(payload.ImageURL)
if provider == "" {
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
@@ -557,13 +560,20 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
logger.Warnf(ctx, "[ImageMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr)
} else if kb != nil {
provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider()))
if backendID == "" && kb.StorageBackendID != nil {
backendID = *kb.StorageBackendID
}
}
}
if s.storageResolver == nil {
return s.fileSvc
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
logger.Infof(ctx, "[ImageMultimodal] resolving file service: tenant=%d provider=%q LOCAL_STORAGE_BASE_DIR=%q imageURL=%s",
payload.TenantID, provider, baseDir, payload.ImageURL)
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
fileSvc, _, svcErr := s.storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir)
if svcErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] resolve file service failed (falling back to default): tenant=%d provider=%s err=%v",
payload.TenantID, provider, svcErr)
@@ -55,6 +55,7 @@ type knowledgeService struct {
tagRepo interfaces.KnowledgeTagRepository
tagService interfaces.KnowledgeTagService
fileSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
modelService interfaces.ModelService
task interfaces.TaskEnqueuer
taskInspector interfaces.TaskInspector
@@ -96,6 +97,7 @@ func NewKnowledgeService(
tagRepo interfaces.KnowledgeTagRepository,
tagService interfaces.KnowledgeTagService,
fileSvc interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
modelService interfaces.ModelService,
task interfaces.TaskEnqueuer,
taskInspector interfaces.TaskInspector,
@@ -122,6 +124,7 @@ func NewKnowledgeService(
tagRepo: tagRepo,
tagService: tagService,
fileSvc: fileSvc,
storageResolver: storageResolver,
modelService: modelService,
task: task,
taskInspector: taskInspector,
@@ -142,15 +142,18 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
}
switch provider {
case "cos":
concreteBackend := kb.StorageBackendID != nil && strings.TrimSpace(*kb.StorageBackendID) != ""
switch {
case concreteBackend:
// Registration already validated and tested the concrete instance.
case provider == "cos":
if tenant == nil || tenant.StorageEngineConfig == nil || tenant.StorageEngineConfig.COS == nil ||
tenant.StorageEngineConfig.COS.SecretID == "" || tenant.StorageEngineConfig.COS.SecretKey == "" ||
tenant.StorageEngineConfig.COS.Region == "" || tenant.StorageEngineConfig.COS.BucketName == "" {
logger.Error(ctx, "COS configuration incomplete for image multimodal processing")
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
}
case "minio":
case provider == "minio":
ok := false
if tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
m := tenant.StorageEngineConfig.MinIO
@@ -235,6 +235,11 @@ func mergeExtractConfig(base types.ExtractConfig, override *types.ExtractConfig)
}
func validateImageMultimodalConfig(ctx context.Context, kb *types.KnowledgeBase) error {
// Concrete backends are validated and connectivity-tested when registered.
// The checks below only apply to unmigrated provider-only bindings.
if kb != nil && kb.StorageBackendID != nil && strings.TrimSpace(*kb.StorageBackendID) != "" {
return nil
}
provider := kb.GetStorageProvider()
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
+50 -2
View File
@@ -115,6 +115,19 @@ func (s *knowledgeService) getVLMConfig(ctx context.Context, kb *types.Knowledge
func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.KnowledgeBase) *types.DocParserStorageConfig {
provider := kb.GetStorageProvider()
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
backendID := ""
if kb.StorageBackendID != nil {
backendID = *kb.StorageBackendID
}
if s.storageResolver != nil && tenant != nil {
if backend, err := s.storageResolver.ResolveBackend(ctx, tenant, backendID, provider); err == nil && backend != nil {
provider = backend.Provider
tenantCopy := *tenant
tenantCopy.StorageEngineConfig = backend.ToStorageEngineConfig()
tenant = &tenantCopy
}
}
if provider == "" {
provider = "local"
}
@@ -131,7 +144,7 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
hasKBFull = sc.SecretID != "" && sc.BucketName != ""
case "minio":
hasKBFull = sc.BucketName != ""
case "local", "tos", "s3", "oss", "ks3":
case "local", "tos", "s3", "oss", "ks3", "obs":
hasKBFull = false
}
@@ -153,7 +166,6 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
var out types.DocParserStorageConfig
out.Provider = strings.ToUpper(provider)
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant != nil && tenant.StorageEngineConfig != nil {
sec := tenant.StorageEngineConfig
if sec.DefaultProvider != "" && provider == "" {
@@ -228,6 +240,15 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
out.BucketName = sec.KS3.BucketName
out.PathPrefix = sec.KS3.PathPrefix
}
case "obs":
if sec.OBS != nil {
out.Endpoint = sec.OBS.Endpoint
out.Region = sec.OBS.Region
out.AccessKeyID = sec.OBS.AccessKey
out.SecretAccessKey = sec.OBS.SecretKey
out.BucketName = sec.OBS.BucketName
out.PathPrefix = sec.OBS.PathPrefix
}
}
}
@@ -248,6 +269,21 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
provider := kb.GetStorageProvider()
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
backendID := ""
if kb.StorageBackendID != nil {
backendID = strings.TrimSpace(*kb.StorageBackendID)
}
if s.storageResolver != nil && tenant != nil {
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
svc, resolvedProvider, err := s.storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir)
if err == nil && svc != nil {
logger.Infof(ctx, "[storage] resolveFileService selected instance: kb=%s backend=%s provider=%s", kb.ID, backendID, resolvedProvider)
return svc
}
if err != nil {
logger.Errorf(ctx, "Failed to resolve storage backend for kb=%s: %v", kb.ID, err)
}
}
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
}
@@ -274,6 +310,18 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
// the provider inferred from the file path. This protects historical data when
// tenant/KB config changes but files were stored under the old provider.
func (s *knowledgeService) resolveFileServiceForPath(ctx context.Context, kb *types.KnowledgeBase, filePath string) interfaces.FileService {
if backendID, inner, ok := types.ParseStorageBackendPath(filePath); ok && s.storageResolver != nil {
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant != nil {
provider := types.ParseProviderScheme(inner)
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
if resolved, _, err := s.storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir); err == nil {
return resolved
} else {
logger.Warnf(ctx, "[storage] failed to resolve backend from file path: backend=%s err=%v", backendID, err)
}
}
}
svc := s.resolveFileService(ctx, kb)
if filePath == "" {
return svc
+81 -46
View File
@@ -27,21 +27,22 @@ var ErrInvalidTenantID = errors.New("invalid tenant ID")
// knowledgeBaseService implements the knowledge base service interface
type knowledgeBaseService struct {
repo interfaces.KnowledgeBaseRepository
kgRepo interfaces.KnowledgeRepository
chunkRepo interfaces.ChunkRepository
shareRepo interfaces.KBShareRepository
kbShareService interfaces.KBShareService
modelService interfaces.ModelService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
tenantRepo interfaces.TenantRepository
fileSvc interfaces.FileService
graphEngine interfaces.RetrieveGraphRepository
asynqClient interfaces.TaskEnqueuer
dsRepo interfaces.DataSourceRepository
syncLogRepo interfaces.SyncLogRepository
dsScheduler *datasource.Scheduler
repo interfaces.KnowledgeBaseRepository
kgRepo interfaces.KnowledgeRepository
chunkRepo interfaces.ChunkRepository
shareRepo interfaces.KBShareRepository
kbShareService interfaces.KBShareService
modelService interfaces.ModelService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
tenantRepo interfaces.TenantRepository
fileSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
graphEngine interfaces.RetrieveGraphRepository
asynqClient interfaces.TaskEnqueuer
dsRepo interfaces.DataSourceRepository
syncLogRepo interfaces.SyncLogRepository
dsScheduler *datasource.Scheduler
}
// NewKnowledgeBaseService creates a new knowledge base service
@@ -55,6 +56,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
ownership retriever.TenantStoreOwnership,
tenantRepo interfaces.TenantRepository,
fileSvc interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
graphEngine interfaces.RetrieveGraphRepository,
asynqClient interfaces.TaskEnqueuer,
dsRepo interfaces.DataSourceRepository,
@@ -62,21 +64,22 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
dsScheduler *datasource.Scheduler,
) interfaces.KnowledgeBaseService {
return &knowledgeBaseService{
repo: repo,
kgRepo: kgRepo,
chunkRepo: chunkRepo,
shareRepo: shareRepo,
kbShareService: kbShareService,
modelService: modelService,
retrieveEngine: retrieveEngine,
ownership: ownership,
tenantRepo: tenantRepo,
fileSvc: fileSvc,
graphEngine: graphEngine,
asynqClient: asynqClient,
dsRepo: dsRepo,
syncLogRepo: syncLogRepo,
dsScheduler: dsScheduler,
repo: repo,
kgRepo: kgRepo,
chunkRepo: chunkRepo,
shareRepo: shareRepo,
kbShareService: kbShareService,
modelService: modelService,
retrieveEngine: retrieveEngine,
ownership: ownership,
tenantRepo: tenantRepo,
fileSvc: fileSvc,
storageResolver: storageResolver,
graphEngine: graphEngine,
asynqClient: asynqClient,
dsRepo: dsRepo,
syncLogRepo: syncLogRepo,
dsScheduler: dsScheduler,
}
}
@@ -119,6 +122,9 @@ func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
}
kb.EnsureDefaults()
applyTenantDefaultStorageProvider(ctx, kb)
if err := s.applyAndValidateStorageBackend(ctx, kb); err != nil {
return nil, err
}
// Fold empty-string vector_store_id into nil so this path and the
// retrieve-engine factory's pre-condition share a single representation.
@@ -150,6 +156,37 @@ func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
return kb, nil
}
func (s *knowledgeBaseService) applyAndValidateStorageBackend(ctx context.Context, kb *types.KnowledgeBase) error {
if s.storageResolver == nil || kb == nil {
return nil
}
tenant, _ := types.TenantInfoFromContext(ctx)
if tenant == nil {
return apperrors.NewBadRequestError("workspace context missing")
}
id := ""
if kb.StorageBackendID != nil {
id = strings.TrimSpace(*kb.StorageBackendID)
}
provider := kb.GetStorageProvider()
// A newly created KB without an explicit instance follows the concrete
// tenant default. The legacy provider is only a fallback for workspaces
// that have not been migrated yet.
if id == "" && tenant.DefaultStorageBackendID != nil && strings.TrimSpace(*tenant.DefaultStorageBackendID) != "" {
provider = ""
}
backend, err := s.storageResolver.ResolveBackend(ctx, tenant, id, provider)
if err != nil {
return apperrors.NewBadRequestError("storage backend is unavailable").WithDetails(err.Error())
}
if backend == nil {
return nil
}
kb.StorageBackendID = &backend.ID
kb.SetStorageProvider(backend.Provider)
return nil
}
// applyTenantDefaultStorageProvider fills an empty KB storage provider from the
// tenant's global default (Settings → Storage engine). Frontend should send the
// same value; this keeps API clients and legacy UIs consistent.
@@ -978,22 +1015,19 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context,
"cross-store cloning is not yet supported")
}
// Defense 3: storage backend must match — only meaningful when the
// tenant has a StorageEngineConfig. Without it, resolveFileService
// ignores per-KB provider pins and routes ALL KBs to the global
// storage service, so a clone can never span two real backends and
// the pins must NOT be used to reject (that would be a false positive).
// When a tenant config exists, pins are honored, so compare effective
// providers and reject a genuine cross-backend clone up front (it would
// otherwise fail mid-clone with ErrCrossBackendCopy).
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil && tenant.StorageEngineConfig != nil {
tenantDefault := tenant.StorageEngineConfig.DefaultProvider
srcProvider := sourceKB.EffectiveStorageProvider(tenantDefault)
dstProvider := targetKB.EffectiveStorageProvider(tenantDefault)
if srcProvider != "" && dstProvider != "" && srcProvider != dstProvider {
return nil, nil, apperrors.NewBadRequestError(fmt.Sprintf(
"source and target knowledge bases use different storage backends (%s vs %s); "+
"cross-storage-backend cloning is not supported", srcProvider, dstProvider))
// Defense 3: the concrete storage instance must match. Comparing only
// provider names would incorrectly allow COS-A -> COS-B clones.
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil {
defaultID, defaultProvider := "", ""
if tenant.DefaultStorageBackendID != nil {
defaultID = *tenant.DefaultStorageBackendID
}
if tenant.StorageEngineConfig != nil {
defaultProvider = tenant.StorageEngineConfig.DefaultProvider
}
if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) {
return nil, nil, apperrors.NewBadRequestError(
"source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported")
}
}
} else {
@@ -1016,6 +1050,7 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context,
SummaryModelID: sourceKB.SummaryModelID,
VLMConfig: sourceKB.VLMConfig,
StorageProviderConfig: sourceKB.StorageProviderConfig,
StorageBackendID: sourceKB.StorageBackendID,
StorageConfig: sourceKB.StorageConfig,
FAQConfig: faqConfig,
VectorStoreID: sourceKB.VectorStoreID,
@@ -0,0 +1,280 @@
package service
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
apperrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type StorageBackendService struct {
repo interfaces.StorageBackendRepository
db *gorm.DB
}
func NewStorageBackendService(repo interfaces.StorageBackendRepository, db *gorm.DB) *StorageBackendService {
return &StorageBackendService{repo: repo, db: db}
}
func (s *StorageBackendService) Create(ctx context.Context, backend *types.StorageBackend) error {
if err := backend.Validate(); err != nil {
return err
}
if err := validateStorageBackendEndpoint(backend); err != nil {
return err
}
if err := s.Test(ctx, backend); err != nil {
return apperrors.NewBadRequestError("storage connection test failed").WithDetails(err.Error())
}
backend.CreatedAt, backend.UpdatedAt = time.Now(), time.Now()
if err := s.repo.Create(ctx, backend); err != nil {
if strings.Contains(strings.ToLower(err.Error()), "unique") {
return apperrors.NewConflictError("a storage backend with this name already exists")
}
return err
}
return nil
}
func (s *StorageBackendService) Update(ctx context.Context, incoming *types.StorageBackend) error {
existing, err := s.repo.GetByID(ctx, incoming.TenantID, incoming.ID)
if err != nil {
return err
}
if existing == nil {
return apperrors.NewNotFoundError("storage backend not found")
}
if existing.Source == types.StorageBackendSourceEnv {
return apperrors.NewBadRequestError("environment storage backend is read-only")
}
incoming.Provider = existing.Provider
incoming.Config = incoming.Config.MergeSecrets(existing.Config)
if incoming.Config.LocationKey(existing.Provider) != existing.Config.LocationKey(existing.Provider) {
return apperrors.NewBadRequestError("endpoint, region, bucket and path prefix are immutable; use storage migration instead")
}
if incoming.Status == "" {
incoming.Status = existing.Status
}
if incoming.Status == types.StorageBackendStatusDisabled && existing.Status != types.StorageBackendStatusDisabled {
var references int64
if err := s.db.WithContext(ctx).Model(&types.Tenant{}).Where("id = ? AND default_storage_backend_id = ?", incoming.TenantID, incoming.ID).Count(&references).Error; err != nil {
return err
}
if references == 0 {
if err := s.db.WithContext(ctx).Model(&types.KnowledgeBase{}).Where("tenant_id = ? AND storage_backend_id = ?", incoming.TenantID, incoming.ID).Count(&references).Error; err != nil {
return err
}
}
if references > 0 {
return apperrors.NewBadRequestError("a default or bound storage backend cannot be disabled")
}
}
if err := incoming.Validate(); err != nil {
return err
}
if err := validateStorageBackendEndpoint(incoming); err != nil {
return err
}
if err := s.Test(ctx, incoming); err != nil {
return apperrors.NewBadRequestError("storage connection test failed").WithDetails(err.Error())
}
incoming.UpdatedAt = time.Now()
return s.repo.Update(ctx, incoming)
}
func (s *StorageBackendService) Delete(ctx context.Context, tenantID uint64, id string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var backend types.StorageBackend
query := tx.Where("tenant_id = ? AND id = ?", tenantID, id)
if tx.Dialector.Name() == "postgres" {
query = query.Clauses(clause.Locking{Strength: "UPDATE"})
}
if err := query.First(&backend).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return apperrors.NewNotFoundError("storage backend not found")
}
return err
}
if backend.Source == types.StorageBackendSourceEnv {
return apperrors.NewBadRequestError("environment storage backend is read-only")
}
var defaultCount int64
if err := tx.Model(&types.Tenant{}).Where("id = ? AND default_storage_backend_id = ?", tenantID, id).Count(&defaultCount).Error; err != nil {
return err
}
if defaultCount > 0 {
return apperrors.NewBadRequestError("default storage backend cannot be deleted")
}
var kbCount int64
if err := tx.Model(&types.KnowledgeBase{}).Where("tenant_id = ? AND storage_backend_id = ?", tenantID, id).Count(&kbCount).Error; err != nil {
return err
}
if kbCount > 0 {
return apperrors.NewBadRequestError(fmt.Sprintf("storage backend still has %d knowledge base(s) bound to it", kbCount))
}
if backend.LegacyAlias {
return apperrors.NewBadRequestError("legacy storage backend cannot be deleted while old file paths may reference it")
}
return tx.Delete(&backend).Error
})
}
func (s *StorageBackendService) SetDefault(ctx context.Context, tenantID uint64, id string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var backend types.StorageBackend
query := tx.Where("tenant_id = ? AND id = ?", tenantID, id)
if tx.Dialector.Name() == "postgres" {
query = query.Clauses(clause.Locking{Strength: "UPDATE"})
}
if err := query.First(&backend).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return apperrors.NewNotFoundError("storage backend not found")
}
return err
}
if backend.Status != types.StorageBackendStatusActive {
return apperrors.NewBadRequestError("only an active storage backend can be the default")
}
return tx.Model(&types.Tenant{}).Where("id = ?", tenantID).Update("default_storage_backend_id", id).Error
})
}
func (s *StorageBackendService) Test(ctx context.Context, backend *types.StorageBackend) error {
if err := backend.Validate(); err != nil {
return err
}
if err := validateStorageBackendEndpoint(backend); err != nil {
return err
}
if backend.Provider == "local" {
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
if baseDir == "" {
baseDir = "/data/files"
}
candidate := filepath.Join(baseDir, strings.Trim(strings.TrimSpace(backend.Config.PathPrefix), "/\\"))
safeDir, err := secutils.SafePathUnderBase(baseDir, candidate)
if err != nil {
return err
}
if err := os.MkdirAll(safeDir, 0o755); err != nil {
return fmt.Errorf("create local storage directory: %w", err)
}
}
c := backend.Config
switch backend.Provider {
case "local":
fileService, _, err := filesvc.NewFileServiceFromStorageConfig("local", backend.ToStorageEngineConfig(), "")
if err != nil {
return err
}
return fileService.CheckConnectivity(ctx)
case "minio":
if c.Mode == "docker" {
c.Endpoint = os.Getenv("MINIO_ENDPOINT")
c.AccessKeyID = os.Getenv("MINIO_ACCESS_KEY_ID")
c.SecretAccessKey = os.Getenv("MINIO_SECRET_ACCESS_KEY")
if c.BucketName == "" {
c.BucketName = os.Getenv("MINIO_BUCKET_NAME")
}
}
return filesvc.CheckMinioConnectivity(ctx, c.Endpoint, c.AccessKeyID, c.SecretAccessKey, c.BucketName, c.UseSSL)
case "cos":
return filesvc.CheckCosConnectivity(ctx, c.BucketName, c.Region, c.AccessKeyID, c.SecretAccessKey)
case "tos":
return filesvc.CheckTosConnectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
case "s3":
return filesvc.CheckS3ConnectivityWithOptions(ctx, c.Endpoint, c.AccessKeyID, c.SecretAccessKey, c.BucketName, c.Region, c.ForcePathStyle)
case "oss":
return filesvc.CheckOssConnectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
case "ks3":
return filesvc.CheckKS3Connectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
case "obs":
return filesvc.CheckObsConnectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
default:
return fmt.Errorf("unsupported storage provider: %s", backend.Provider)
}
}
func (s *StorageBackendService) ResolveBackend(ctx context.Context, tenant *types.Tenant, backendID, provider string) (*types.StorageBackend, error) {
if tenant == nil {
return nil, fmt.Errorf("workspace context missing")
}
backendID = strings.TrimSpace(backendID)
provider = strings.ToLower(strings.TrimSpace(provider))
if backendID == "" && provider != "" {
backend, err := s.repo.FindLegacyAlias(ctx, tenant.ID, provider)
if err != nil || backend != nil {
return backend, err
}
}
if backendID == "" && tenant.DefaultStorageBackendID != nil {
backendID = strings.TrimSpace(*tenant.DefaultStorageBackendID)
}
if backendID != "" {
backend, err := s.repo.GetByID(ctx, tenant.ID, backendID)
if err != nil {
return nil, err
}
if backend == nil {
return nil, fmt.Errorf("storage backend not found")
}
if backend.Status != types.StorageBackendStatusActive {
return nil, fmt.Errorf("storage backend is not active")
}
return backend, nil
}
return nil, nil
}
func (s *StorageBackendService) ResolveFileService(ctx context.Context, tenant *types.Tenant, backendID, provider, localBaseDir string) (interfaces.FileService, string, error) {
backend, err := s.ResolveBackend(ctx, tenant, backendID, provider)
if err != nil {
return nil, "", err
}
if backend != nil {
inner, provider, err := filesvc.NewFileServiceFromStorageConfig(backend.Provider, backend.ToStorageEngineConfig(), localBaseDir)
if err != nil {
return nil, provider, err
}
return filesvc.NewBackendScopedFileService(backend.ID, inner), provider, nil
}
if tenant == nil {
return nil, "", fmt.Errorf("workspace context missing")
}
return filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, localBaseDir)
}
func validateStorageBackendEndpoint(backend *types.StorageBackend) error {
if backend.Provider == "local" || (backend.Provider == "minio" && backend.Config.Mode == "docker") {
return nil
}
endpoint := strings.TrimSpace(backend.Config.Endpoint)
if backend.Provider == "cos" || endpoint == "" {
return nil
}
if !strings.Contains(endpoint, "://") {
scheme := "https://"
if backend.Provider == "minio" && !backend.Config.UseSSL {
scheme = "http://"
}
endpoint = scheme + endpoint
}
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
return apperrors.NewBadRequestError("storage endpoint failed SSRF validation").WithDetails(err.Error())
}
return nil
}
var _ interfaces.StorageBackendService = (*StorageBackendService)(nil)
var _ interfaces.StorageBackendResolver = (*StorageBackendService)(nil)
@@ -0,0 +1,42 @@
package service_test
import (
"context"
"io"
"testing"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func TestStorageBackendResolverScopesPathsAndTenant(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&types.StorageBackend{}))
repo := repository.NewStorageBackendRepository(db)
backend := &types.StorageBackend{TenantID: 7, Name: "Local A", Provider: "local", Config: types.StorageBackendConfig{}, LegacyAlias: true}
require.NoError(t, repo.Create(context.Background(), backend))
resolver := service.NewStorageBackendService(repo, db)
svc, provider, err := resolver.ResolveFileService(context.Background(), &types.Tenant{ID: 7}, backend.ID, "local", t.TempDir())
require.NoError(t, err)
assert.Equal(t, "local", provider)
path, err := svc.SaveBytes(context.Background(), []byte("scoped"), 7, "exports/a.txt", false)
require.NoError(t, err)
assert.Contains(t, path, "storage://"+backend.ID+"/local://")
reader, err := svc.GetFile(context.Background(), path)
require.NoError(t, err)
defer reader.Close()
data, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "scoped", string(data))
_, _, err = resolver.ResolveFileService(context.Background(), &types.Tenant{ID: 8}, backend.ID, "local", t.TempDir())
require.Error(t, err)
}
+37 -3
View File
@@ -21,12 +21,13 @@ type ListTenantsParams struct {
// tenantService implements the TenantService interface
type tenantService struct {
repo interfaces.TenantRepository // Repository for tenant data operations
repo interfaces.TenantRepository // Repository for tenant data operations
storageRepo interfaces.StorageBackendRepository
}
// NewTenantService creates a new tenant service instance
func NewTenantService(repo interfaces.TenantRepository) interfaces.TenantService {
return &tenantService{repo: repo}
func NewTenantService(repo interfaces.TenantRepository, storageRepo interfaces.StorageBackendRepository) interfaces.TenantService {
return &tenantService{repo: repo, storageRepo: storageRepo}
}
// CreateTenant creates a new tenant
@@ -60,11 +61,44 @@ func (s *tenantService) CreateTenant(ctx context.Context, tenant *types.Tenant)
})
return nil, err
}
if err := s.createDefaultStorageBackend(ctx, tenant); err != nil {
// No related rows exist yet, so rolling the tenant back is safe and
// avoids leaving a workspace that cannot bind new knowledge bases.
_ = s.repo.DeleteTenant(ctx, tenant.ID)
return nil, err
}
logger.Infof(ctx, "Tenant created successfully, ID: %d, name: %s", tenant.ID, tenant.Name)
return tenant, nil
}
func (s *tenantService) createDefaultStorageBackend(ctx context.Context, tenant *types.Tenant) error {
if s.storageRepo == nil || tenant == nil {
return nil
}
provider := ""
if tenant.StorageEngineConfig != nil {
provider = tenant.StorageEngineConfig.DefaultProvider
}
backend := types.StorageBackendFromLegacy(tenant.ID, provider, tenant.StorageEngineConfig)
if backend == nil {
backend = types.StorageBackendFromEnvironment(tenant.ID)
}
if backend == nil {
return errors.New("no supported default storage backend is configured")
}
backend.LegacyAlias = true
if err := s.storageRepo.Create(ctx, backend); err != nil {
return err
}
tenant.DefaultStorageBackendID = &backend.ID
if err := s.repo.UpdateTenant(ctx, tenant); err != nil {
_ = s.storageRepo.Delete(ctx, tenant.ID, backend.ID)
return err
}
return nil
}
// GetTenantByID retrieves a tenant by their ID
func (s *tenantService) GetTenantByID(ctx context.Context, id uint64) (*types.Tenant, error) {
if id == 0 {
@@ -0,0 +1,35 @@
package service_test
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func TestCreateTenantCreatesConcreteDefaultStorageBackend(t *testing.T) {
t.Setenv("STORAGE_TYPE", "local")
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&types.Tenant{}, &types.StorageBackend{}))
tenantRepo := repository.NewTenantRepository(db)
storageRepo := repository.NewStorageBackendRepository(db)
tenantSvc := service.NewTenantService(tenantRepo, storageRepo)
tenant, err := tenantSvc.CreateTenant(context.Background(), &types.Tenant{Name: "workspace"})
require.NoError(t, err)
require.NotNil(t, tenant.DefaultStorageBackendID)
backend, err := storageRepo.GetByID(context.Background(), tenant.ID, *tenant.DefaultStorageBackendID)
require.NoError(t, err)
require.NotNil(t, backend)
assert.Equal(t, "local", backend.Provider)
assert.Equal(t, types.StorageBackendSourceEnv, backend.Source)
assert.True(t, backend.LegacyAlias)
}
@@ -137,7 +137,7 @@ func mockEngineFactory(err error) interfaces.EngineFactory {
// mockEngineService satisfies interfaces.RetrieveEngineService minimally.
type mockEngineService struct{}
func (m *mockEngineService) EngineType() types.RetrieverEngineType { return "mock" }
func (m *mockEngineService) EngineType() types.RetrieverEngineType { return "mock" }
func (m *mockEngineService) Retrieve(_ context.Context, _ types.RetrieveParams) ([]*types.RetrieveResult, error) {
return nil, nil
}
@@ -842,7 +842,8 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
is_pinned INTEGER NOT NULL DEFAULT 0,
pinned_at DATETIME NULL,
asr_config TEXT,
vector_store_id VARCHAR(36),
vector_store_id VARCHAR(36),
storage_backend_id VARCHAR(36),
wiki_config TEXT,
indexing_strategy TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
@@ -1077,7 +1078,7 @@ func TestDeleteStore_Guard_RejectsCount3(t *testing.T) {
storeID := "store-A"
for i := 0; i < 3; i++ {
kb := &types.KnowledgeBase{
ID: "kb-multi-" + tenantID2s(1) + "-" + ptrOrEmpty(&storeID) + "-" + string(rune('a'+i)),
ID: "kb-multi-" + tenantID2s(1) + "-" + ptrOrEmpty(&storeID) + "-" + string(rune('a'+i)),
Name: "kb", TenantID: 1, EmbeddingModelID: "e", SummaryModelID: "s",
VectorStoreID: &storeID,
}
+104 -2
View File
@@ -81,6 +81,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/limiter"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/router"
"github.com/Tencent/WeKnora/internal/storageallowlist"
"github.com/Tencent/WeKnora/internal/stream"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
@@ -229,6 +230,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Invoke(registerWebSearchProviders))
must(container.Provide(repository.NewWebSearchProviderRepository))
must(container.Provide(repository.NewVectorStoreRepository))
must(container.Provide(repository.NewStorageBackendRepository))
// TenantStoreOwnership adapter used by the retriever factory functions
// to verify that a resolved VectorStore belongs to the caller's tenant.
must(container.Provide(retriever.NewVectorStoreRepoOwnership))
@@ -245,6 +247,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
return sr, nil
}))
must(container.Provide(service.NewVectorStoreService))
must(container.Provide(service.NewStorageBackendService))
must(container.Provide(func(s *service.StorageBackendService) interfaces.StorageBackendService { return s }))
must(container.Provide(func(s *service.StorageBackendService) interfaces.StorageBackendResolver { return s }))
// Agent service layer (requires event bus, web search service)
// SessionService is passed as parameter to CreateAgentEngine method when creating AgentService
@@ -358,6 +363,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Provide(handler.NewWebSearchHandler))
must(container.Provide(handler.NewWebSearchProviderHandler))
must(container.Provide(handler.NewVectorStoreHandler))
must(container.Provide(handler.NewStorageBackendHandler))
must(container.Provide(handler.NewCustomAgentHandler))
must(container.Provide(handler.NewUserResourceFavoriteHandler))
must(container.Provide(service.NewSkillService))
@@ -406,7 +412,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
// disk bytes requires rebuilding the FileService from that tenant's storage
// config. The owning tenant is parsed from the URL's first path segment, which
// correctly handles cross-tenant shared resources (e.g. shared KB images).
func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository) {
func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository, storageResolver interfaces.StorageBackendResolver) {
chat.LocalImageResolver = func(storageURL string) ([]byte, bool) {
tenantID := secutils.ParseTenantIDFromStoragePath(storageURL)
if tenantID == 0 {
@@ -418,7 +424,12 @@ func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository) {
return nil, false
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
fileSvc, _, err := file.NewFileServiceFromStorageConfig("local", tenant.StorageEngineConfig, baseDir)
backendID, inner, _ := types.ParseStorageBackendPath(storageURL)
provider := types.ParseProviderScheme(inner)
if provider == "" {
provider = "local"
}
fileSvc, _, err := storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir)
if err != nil {
return nil, false
}
@@ -665,6 +676,7 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
// The SQL migration marks KBs that have documents but no provider with "__pending_env__";
// we replace that with the actual STORAGE_TYPE from the environment.
resolveStorageProviderPending(db)
migrateLegacyStorageBackends(db)
// Post-migration: declarative built-in models from config/builtin_models.yaml (optional).
if err := types.LoadBuiltinModelsConfig(context.Background(), db, config.ConfigDir()); err != nil {
@@ -723,6 +735,96 @@ func resolveStorageProviderPending(db *gorm.DB) {
resetPendingTasks(db)
}
// migrateLegacyStorageBackends backfills the storage_backends table from each
// workspace's legacy StorageEngineConfig (or environment defaults) and binds
// existing knowledge bases to the resulting backend.
//
// The table, columns and indexes are created by the SQL migrations
// (migrations/versioned/000068 for Postgres, migrations/sqlite/000000_init for
// SQLite); this step only handles data that cannot be expressed portably in
// SQL: environment snapshots, JSON→config mapping, AES-encrypted credentials,
// UUID generation and the per-startup refresh of env-backed aliases.
// The migration is idempotent: one legacy_alias row per tenant/provider.
func migrateLegacyStorageBackends(db *gorm.DB) {
var tenants []*types.Tenant
if err := db.Find(&tenants).Error; err != nil {
logger.Warnf(context.Background(), "Failed to load workspaces for storage backend migration: %v", err)
return
}
for _, tenant := range tenants {
legacy := tenant.StorageEngineConfig
defaultProvider := ""
if legacy != nil {
defaultProvider = strings.ToLower(strings.TrimSpace(legacy.DefaultProvider))
}
if defaultProvider == "" {
defaultProvider = strings.ToLower(strings.TrimSpace(os.Getenv("STORAGE_TYPE")))
}
if defaultProvider == "" {
defaultProvider = "local"
}
backendIDs := make(map[string]string)
for _, provider := range storageallowlist.Supported() {
var existing types.StorageBackend
err := db.Where("tenant_id = ? AND provider = ? AND legacy_alias = ?", tenant.ID, provider, true).First(&existing).Error
if err == nil {
// Environment-backed aliases are snapshots, not user-owned config.
// Refresh them at every startup so credential rotation does not
// leave the persisted resolver on stale values. If the workspace
// later gains an explicit legacy config, promote the alias to user
// source and stop automatic refreshes.
if existing.Source == types.StorageBackendSourceEnv {
desired := types.StorageBackendFromLegacy(tenant.ID, provider, legacy)
if desired == nil && provider == defaultProvider {
desired = types.StorageBackendFromEnvironment(tenant.ID)
}
if desired != nil && desired.Provider == provider {
_ = db.Model(&types.StorageBackend{}).Where("id = ?", existing.ID).Updates(map[string]interface{}{
"name": desired.Name, "config": desired.Config, "source": desired.Source, "status": desired.Status, "updated_at": time.Now(),
}).Error
}
}
backendIDs[provider] = existing.ID
continue
}
backend := types.StorageBackendFromLegacy(tenant.ID, provider, legacy)
if backend == nil && provider == defaultProvider {
backend = types.StorageBackendFromEnvironment(tenant.ID)
}
if backend == nil {
continue
}
if err := db.Create(backend).Error; err != nil {
logger.Warnf(context.Background(), "Failed to migrate %s storage for workspace %d: %v", provider, tenant.ID, err)
continue
}
backendIDs[provider] = backend.ID
}
if tenant.DefaultStorageBackendID == nil {
if id := backendIDs[defaultProvider]; id != "" {
if err := db.Model(&types.Tenant{}).Where("id = ?", tenant.ID).Update("default_storage_backend_id", id).Error; err != nil {
logger.Warnf(context.Background(), "Failed to set default storage backend for workspace %d: %v", tenant.ID, err)
}
}
}
var kbs []*types.KnowledgeBase
if err := db.Where("tenant_id = ? AND storage_backend_id IS NULL", tenant.ID).Find(&kbs).Error; err != nil {
continue
}
for _, kb := range kbs {
provider := kb.GetStorageProvider()
if provider == "" {
provider = defaultProvider
}
if id := backendIDs[provider]; id != "" {
_ = db.Model(&types.KnowledgeBase{}).Where("id = ? AND storage_backend_id IS NULL", kb.ID).Update("storage_backend_id", id).Error
}
}
}
}
// syncSequences ensures PostgreSQL sequences for auto-increment columns (seq_id)
// are at least as high as the current MAX value in each table. This is needed
// because older code assigned seq_id via application-level MAX()+1, which could
@@ -0,0 +1,45 @@
package container
import (
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func TestMigrateLegacyStorageBackends(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
// Schema is owned by the SQL migrations in production; provision it here so
// the test exercises only the data-migration logic.
require.NoError(t, db.Exec(`CREATE TABLE tenants (
id INTEGER PRIMARY KEY, name TEXT, storage_engine_config TEXT,
default_storage_backend_id TEXT, updated_at DATETIME, deleted_at DATETIME
)`).Error)
require.NoError(t, db.Exec(`CREATE TABLE knowledge_bases (
id TEXT PRIMARY KEY, tenant_id INTEGER, storage_provider_config TEXT,
storage_backend_id TEXT, cos_config TEXT, updated_at DATETIME, deleted_at DATETIME
)`).Error)
require.NoError(t, db.AutoMigrate(&types.StorageBackend{}))
tenantConfig, err := (&types.StorageEngineConfig{DefaultProvider: "local", Local: &types.LocalEngineConfig{PathPrefix: "workspace-a"}}).Value()
require.NoError(t, err)
providerConfig, err := (types.StorageProviderConfig{Provider: "local"}).Value()
require.NoError(t, err)
require.NoError(t, db.Exec("INSERT INTO tenants(id, name, storage_engine_config) VALUES (?, ?, ?)", 7, "workspace", tenantConfig).Error)
require.NoError(t, db.Exec("INSERT INTO knowledge_bases(id, tenant_id, storage_provider_config, cos_config) VALUES (?, ?, ?, ?)", "kb-a", 7, providerConfig, []byte("{}")).Error)
migrateLegacyStorageBackends(db)
var backend types.StorageBackend
require.NoError(t, db.Where("tenant_id = ? AND provider = ? AND legacy_alias = ?", 7, "local", true).First(&backend).Error)
assert.Equal(t, "workspace-a", backend.Config.PathPrefix)
var tenantDefault, kbBackend string
require.NoError(t, db.Raw("SELECT default_storage_backend_id FROM tenants WHERE id = 7").Scan(&tenantDefault).Error)
require.NoError(t, db.Raw("SELECT storage_backend_id FROM knowledge_bases WHERE id = 'kb-a'").Scan(&kbBackend).Error)
assert.Equal(t, backend.ID, tenantDefault)
assert.Equal(t, backend.ID, kbBackend)
}
+29 -2
View File
@@ -63,6 +63,7 @@ type InitializationHandler struct {
ollamaService *ollama.OllamaService
documentReader interfaces.DocumentReader
pooler embedding.EmbedderPooler
storageResolver interfaces.StorageBackendResolver
}
// NewInitializationHandler 创建初始化处理器
@@ -76,6 +77,7 @@ func NewInitializationHandler(
ollamaService *ollama.OllamaService,
documentReader interfaces.DocumentReader,
pooler embedding.EmbedderPooler,
storageResolver interfaces.StorageBackendResolver,
) *InitializationHandler {
return &InitializationHandler{
config: config,
@@ -87,6 +89,7 @@ func NewInitializationHandler(
ollamaService: ollamaService,
documentReader: documentReader,
pooler: pooler,
storageResolver: storageResolver,
}
}
@@ -123,7 +126,8 @@ type KBModelConfigRequest struct {
} `json:"multimodal"`
// 存储引擎选择("local" | "minio" | "cos"),影响文档上传与文档内图片存储,参数从全局设置读取
StorageProvider string `json:"storageProvider"`
StorageProvider string `json:"storageProvider"`
StorageBackendID string `json:"storageBackendId"`
// 知识图谱配置
NodeExtract struct {
@@ -362,7 +366,30 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
kb.VLMConfig.CustomInstructions = strings.TrimSpace(req.VLMConfig.CustomInstructions)
}
// 存储引擎:仅写入 provider 到新字段,参数从空间全局 StorageEngineConfig 读取
// Bind the concrete storage instance. Provider remains a compatibility
// projection for older clients and historical rows.
if strings.TrimSpace(req.StorageBackendID) != "" {
tenant, _ := types.TenantInfoFromContext(ctx)
backend, resolveErr := h.storageResolver.ResolveBackend(ctx, tenant, req.StorageBackendID, "")
if resolveErr != nil || backend == nil {
c.Error(errors.NewBadRequestError("Storage backend is unavailable"))
return
}
oldID := ""
if kb.StorageBackendID != nil {
oldID = *kb.StorageBackendID
}
if oldID != "" && oldID != backend.ID {
knowledgeList, listErr := h.knowledgeService.ListPagedKnowledgeByKnowledgeBaseID(ctx, kbIdStr, &types.Pagination{Page: 1, PageSize: 1}, types.KnowledgeListFilter{})
if listErr == nil && knowledgeList != nil && knowledgeList.Total > 0 {
c.Error(errors.NewBadRequestError("Storage backend cannot be changed while the knowledge base contains files; migrate storage first"))
return
}
}
kb.StorageBackendID = &backend.ID
req.StorageProvider = backend.Provider
}
// Legacy provider projection.
provider := strings.ToLower(strings.TrimSpace(req.StorageProvider))
if provider == "" {
provider = "local"
+12 -14
View File
@@ -1055,21 +1055,19 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
"cross-store cloning is not yet supported"))
return
}
// Pre-flight defense 3: storage backend must match — only meaningful
// when the tenant has a StorageEngineConfig. Without it,
// resolveFileService ignores per-KB provider pins and routes ALL KBs to
// the global storage service, so a clone can never span two real
// backends and the pins must NOT be used to reject (false positive).
// When a tenant config exists, pins are honored, so reject a genuine
// cross-backend clone before enqueueing.
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil && tenant.StorageEngineConfig != nil {
tenantDefault := tenant.StorageEngineConfig.DefaultProvider
srcProvider := sourceKB.EffectiveStorageProvider(tenantDefault)
dstProvider := targetKB.EffectiveStorageProvider(tenantDefault)
if srcProvider != "" && dstProvider != "" && srcProvider != dstProvider {
// Pre-flight defense 3: compare concrete instance IDs, not just the
// provider type (COS-A and COS-B are different physical stores).
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil {
defaultID, defaultProvider := "", ""
if tenant.DefaultStorageBackendID != nil {
defaultID = *tenant.DefaultStorageBackendID
}
if tenant.StorageEngineConfig != nil {
defaultProvider = tenant.StorageEngineConfig.DefaultProvider
}
if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) {
c.Error(apperrors.NewBadRequestError(
"source and target knowledge bases use different storage backends (" +
srcProvider + " vs " + dstProvider + "); cross-storage-backend cloning is not supported"))
"source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported"))
return
}
}
+6 -3
View File
@@ -27,9 +27,10 @@ type Handler struct {
agentShareService interfaces.AgentShareService // Service for resolving shared agents (KB scope in retrieval)
kbShareService interfaces.KBShareService // Service for resolving shared KB permissions
fileService interfaces.FileService // Service for file storage (image uploads)
modelService interfaces.ModelService // Service for model management (VLM access)
userService interfaces.UserService // Service for resolving per-user preferences (e.g. enable_memory default)
attachmentProcessor *AttachmentProcessor // Processor for file attachments
storageResolver interfaces.StorageBackendResolver
modelService interfaces.ModelService // Service for model management (VLM access)
userService interfaces.UserService // Service for resolving per-user preferences (e.g. enable_memory default)
attachmentProcessor *AttachmentProcessor // Processor for file attachments
}
// NewHandler creates a new instance of Handler with all necessary dependencies
@@ -45,6 +46,7 @@ func NewHandler(
agentShareService interfaces.AgentShareService,
kbShareService interfaces.KBShareService,
fileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
modelService interfaces.ModelService,
userService interfaces.UserService,
documentReader interfaces.DocumentReader,
@@ -62,6 +64,7 @@ func NewHandler(
agentShareService: agentShareService,
kbShareService: kbShareService,
fileService: fileService,
storageResolver: storageResolver,
modelService: modelService,
userService: userService,
attachmentProcessor: NewAttachmentProcessor(
+13 -4
View File
@@ -141,12 +141,21 @@ func mimeToExt(mime string) string {
}
func (h *Handler) resolveImageFileService(ctx context.Context, storageProvider string) interfaces.FileService {
if strings.TrimSpace(storageProvider) == "" {
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
return h.fileService
}
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil || tenant.StorageEngineConfig == nil {
if h.storageResolver != nil {
svc, resolvedProvider, err := h.storageResolver.ResolveFileService(ctx, tenant, "", storageProvider, "")
if err == nil && svc != nil {
logger.Infof(ctx, "[image-storage] using storage instance provider=%s for image uploads", resolvedProvider)
return svc
}
if err != nil {
logger.Warnf(ctx, "[image-storage] failed to resolve storage instance for provider=%s: %v", storageProvider, err)
}
}
if strings.TrimSpace(storageProvider) == "" || tenant.StorageEngineConfig == nil {
return h.fileService
}
+149
View File
@@ -0,0 +1,149 @@
package handler
import (
"net/http"
apperrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/storageallowlist"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/gin-gonic/gin"
)
type StorageBackendHandler struct {
repo interfaces.StorageBackendRepository
service interfaces.StorageBackendService
}
func NewStorageBackendHandler(repo interfaces.StorageBackendRepository, service interfaces.StorageBackendService) *StorageBackendHandler {
return &StorageBackendHandler{repo: repo, service: service}
}
type storageBackendRequest struct {
Name string `json:"name" binding:"required"`
Provider string `json:"provider" binding:"required"`
Config types.StorageBackendConfig `json:"config"`
Status string `json:"status,omitempty"`
}
func storageTenantID(c *gin.Context) uint64 { return c.GetUint64(types.TenantIDContextKey.String()) }
func (h *StorageBackendHandler) List(c *gin.Context) {
tenantID := storageTenantID(c)
backends, err := h.repo.List(c.Request.Context(), tenantID)
if err != nil {
c.Error(err)
return
}
result := make([]types.StorageBackend, 0, len(backends))
for _, backend := range backends {
result = append(result, types.NewStorageBackendResponse(backend))
}
tenant, _ := types.TenantInfoFromContext(c.Request.Context())
var defaultID *string
if tenant != nil {
defaultID = tenant.DefaultStorageBackendID
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": result, "default_storage_backend_id": defaultID})
}
func (h *StorageBackendHandler) Get(c *gin.Context) {
backend, err := h.repo.GetByID(c.Request.Context(), storageTenantID(c), c.Param("id"))
if err != nil {
c.Error(err)
return
}
if backend == nil {
c.Error(apperrors.NewNotFoundError("storage backend not found"))
return
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": types.NewStorageBackendResponse(backend)})
}
func (h *StorageBackendHandler) Create(c *gin.Context) {
var req storageBackendRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.Error(apperrors.NewBadRequestError(err.Error()))
return
}
backend := &types.StorageBackend{TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config, Status: req.Status}
if err := h.service.Create(c.Request.Context(), backend); err != nil {
c.Error(err)
return
}
c.JSON(http.StatusCreated, gin.H{"success": true, "data": types.NewStorageBackendResponse(backend)})
}
func (h *StorageBackendHandler) Update(c *gin.Context) {
var req storageBackendRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.Error(apperrors.NewBadRequestError(err.Error()))
return
}
backend := &types.StorageBackend{ID: c.Param("id"), TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config, Status: req.Status}
if err := h.service.Update(c.Request.Context(), backend); err != nil {
c.Error(err)
return
}
updated, err := h.repo.GetByID(c.Request.Context(), backend.TenantID, backend.ID)
if err != nil {
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": types.NewStorageBackendResponse(updated)})
}
func (h *StorageBackendHandler) Delete(c *gin.Context) {
if err := h.service.Delete(c.Request.Context(), storageTenantID(c), c.Param("id")); err != nil {
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{"success": true})
}
func (h *StorageBackendHandler) SetDefault(c *gin.Context) {
if err := h.service.SetDefault(c.Request.Context(), storageTenantID(c), c.Param("id")); err != nil {
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{"success": true})
}
func (h *StorageBackendHandler) TestRaw(c *gin.Context) {
var req storageBackendRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.Error(apperrors.NewBadRequestError(err.Error()))
return
}
backend := &types.StorageBackend{TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config}
if err := backend.Validate(); err != nil {
c.Error(err)
return
}
if err := h.service.Test(c.Request.Context(), backend); err != nil {
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"success": true})
}
func (h *StorageBackendHandler) TestByID(c *gin.Context) {
backend, err := h.repo.GetByID(c.Request.Context(), storageTenantID(c), c.Param("id"))
if err != nil {
c.Error(err)
return
}
if backend == nil {
c.Error(apperrors.NewNotFoundError("storage backend not found"))
return
}
if err := h.service.Test(c.Request.Context(), backend); err != nil {
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"success": true})
}
func (h *StorageBackendHandler) Types(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"success": true, "data": storageallowlist.AllowedList()})
}
+10
View File
@@ -98,6 +98,11 @@ func TestFindIncompleteStorageURL(t *testing.T) {
"text minio://",
5,
},
{
"partial scoped URL at end",
"text storage://backend-a/co",
5,
},
{
"no storage URL",
"just plain text http://example.com",
@@ -291,6 +296,11 @@ func TestHoldbackCutoff(t *testing.T) {
"prefix minio://wizard-test/10000/exp",
7,
},
{
"bare truncated scoped URL without markdown wrapper",
"prefix storage://backend-a/cos://bucket/10000/exp",
7,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
+12
View File
@@ -111,6 +111,18 @@ func TestRewriteStorageURLs_MinIOFallbackViaGlobal(t *testing.T) {
assert.NotContains(t, out, "](minio://")
}
func TestRewriteStorageURLs_ScopedPath(t *testing.T) {
stub := &stubIMFileService{
getFileURL: func(_ context.Context, filePath string) (string, error) {
assert.Equal(t, "storage://backend-a/cos://bucket/ap-test/10000/exports/a.png", filePath)
return "https://storage.example/a.png", nil
},
}
input := "![img](storage://backend-a/cos://bucket/ap-test/10000/exports/a.png)"
output := rewriteStorageURLs(context.Background(), input, newIMFileServiceResolver(&types.Tenant{}, stub))
assert.Contains(t, output, "https://storage.example/a.png")
}
func TestCleanIMContent_MinIOFallbackIntegration(t *testing.T) {
stub := &stubIMFileService{
getFileURL: func(_ context.Context, _ string) (string, error) {
+41 -19
View File
@@ -77,8 +77,9 @@ func stripImageXMLTags(s string) string {
})
}
// storageSchemeRe matches provider:// URLs used by file storage backends.
var storageSchemeRe = regexp.MustCompile(`\b(local|minio|s3|cos|tos|oss)://[^\s)\]>"]+`)
// storageSchemeRe matches both legacy provider:// URLs and canonical
// storage://<backend-id>/provider:// URLs.
var storageSchemeRe = regexp.MustCompile(`\b(?:storage://[0-9A-Za-z_-]+/)?(?:local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]+`)
// rewriteStorageURLs replaces all provider:// URLs in content with HTTP URLs
// obtained from fileService.GetFileURL. URLs that are already HTTP or cannot
@@ -127,7 +128,7 @@ func rewriteStorageURLs(ctx context.Context, content string, resolver *imFileSer
// incompleteURLSuffixRe matches a provider:// URL that reaches the end of the
// string — it may continue in the next chunk.
var incompleteURLSuffixRe = regexp.MustCompile(
`\b(?:local|minio|s3|cos|tos|oss)://[^\s)\]>"]*$`,
`\b(?:storage|local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]*$`,
)
// findIncompleteStorageURL returns the byte offset of a potentially truncated
@@ -197,18 +198,19 @@ func holdbackCutoff(chunk string) int {
}
// formatIMOutboundAnswer strips thinking/tool blocks and applies IM content cleanup.
func formatIMOutboundAnswer(ctx context.Context, raw string, tenant *types.Tenant, defaultFileSvc interfaces.FileService) string {
return cleanIMContent(ctx, FormatIMDisplayContent(raw, StreamDisplayFinal), tenant, defaultFileSvc)
func formatIMOutboundAnswer(ctx context.Context, raw string, tenant *types.Tenant, defaultFileSvc interfaces.FileService, storageResolvers ...interfaces.StorageBackendResolver) string {
return cleanIMContent(ctx, FormatIMDisplayContent(raw, StreamDisplayFinal), tenant, defaultFileSvc, storageResolvers...)
}
// cleanIMContent applies all IM-specific content transformations:
// 1. Collapse <image> XML blocks back to plain markdown
// 2. Strip <kb/> and <web/> citation tags
// 3. Rewrite provider:// URLs to HTTP URLs (scheme-aware per tenant config)
func cleanIMContent(ctx context.Context, content string, tenant *types.Tenant, defaultFileSvc interfaces.FileService) string {
func cleanIMContent(ctx context.Context, content string, tenant *types.Tenant, defaultFileSvc interfaces.FileService, storageResolvers ...interfaces.StorageBackendResolver) string {
content = stripImageXMLTags(content)
content = stripIMCitationTags(content)
resolver := newIMFileServiceResolver(tenant, defaultFileSvc)
resolver := newIMFileServiceResolver(tenant, defaultFileSvc, storageResolvers...)
resolver.ctx = ctx
content = rewriteStorageURLs(ctx, content, resolver)
return content
}
@@ -225,20 +227,28 @@ func imLocalStorageBaseDir() string {
// for the lifetime of one cleanIMContent / outbound message (avoids re-creating SDK clients
// for every URL in a long answer).
type imFileServiceResolver struct {
tenant *types.Tenant
defaultSvc interfaces.FileService
cache map[string]interfaces.FileService
tenant *types.Tenant
defaultSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
ctx context.Context
cache map[string]interfaces.FileService
}
func newIMFileServiceResolver(tenant *types.Tenant, defaultSvc interfaces.FileService) *imFileServiceResolver {
return &imFileServiceResolver{
func newIMFileServiceResolver(tenant *types.Tenant, defaultSvc interfaces.FileService, storageResolvers ...interfaces.StorageBackendResolver) *imFileServiceResolver {
resolver := &imFileServiceResolver{
tenant: tenant,
defaultSvc: defaultSvc,
ctx: context.Background(),
cache: make(map[string]interfaces.FileService),
}
if len(storageResolvers) > 0 {
resolver.storageResolver = storageResolvers[0]
}
return resolver
}
func (r *imFileServiceResolver) resolve(filePath string) interfaces.FileService {
backendID, _, _ := types.ParseStorageBackendPath(filePath)
provider := types.ParseProviderScheme(filePath)
if provider == "" {
if r.tenant != nil && r.tenant.StorageEngineConfig != nil {
@@ -248,12 +258,21 @@ func (r *imFileServiceResolver) resolve(filePath string) interfaces.FileService
return nil
}
}
if svc, ok := r.cache[provider]; ok {
cacheKey := backendID + ":" + provider
if svc, ok := r.cache[cacheKey]; ok {
return svc
}
if r.storageResolver != nil && r.tenant != nil {
svc, _, err := r.storageResolver.ResolveFileService(r.ctx, r.tenant, backendID, provider, imLocalStorageBaseDir())
if err == nil {
r.cache[cacheKey] = svc
return svc
}
logger.Warnf(r.ctx, "[IM] resolve storage backend failed: backend_id=%s provider=%s err=%v", backendID, provider, err)
}
svc := buildIMFileServiceForProvider(r.tenant, provider, r.defaultSvc)
if svc != nil {
r.cache[provider] = svc
r.cache[cacheKey] = svc
}
return svc
}
@@ -374,7 +393,8 @@ type Service struct {
// defaultFileSvc is the process-wide storage backend (STORAGE_TYPE / env).
// Used when tenant StorageEngineConfig cannot build a service for the URL scheme.
defaultFileSvc interfaces.FileService
defaultFileSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
// cmdRegistry holds all registered slash-commands.
cmdRegistry *CommandRegistry
@@ -787,6 +807,7 @@ func NewService(
oauthManager *mcppkg.OAuthManager,
redisClient *redis.Client,
appCfg *config.Config,
storageResolver interfaces.StorageBackendResolver,
) *Service {
// Resolve IM configuration with defaults.
workers, maxQueue, maxPerUser, globalMaxWorkers, rlWindow, rlMax := resolveIMConfig(appCfg)
@@ -811,6 +832,7 @@ func NewService(
modelService: modelService,
streamManager: streamManager,
defaultFileSvc: defaultFileSvc,
storageResolver: storageResolver,
oauthManager: oauthManager,
cmdRegistry: registry,
channels: make(map[string]*channelState),
@@ -1602,7 +1624,7 @@ func (s *Service) executeQARequest(req *qaRequest) {
}
reply := &ReplyMessage{
Content: formatIMOutboundAnswer(ctx, answer, req.tenant, s.defaultFileSvc),
Content: formatIMOutboundAnswer(ctx, answer, req.tenant, s.defaultFileSvc, s.storageResolver),
IsFinal: true,
}
if err := req.adapter.SendReply(ctx, req.msg, reply); err != nil {
@@ -2357,7 +2379,7 @@ func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage,
displaySource = displaySource[:cut]
}
display := cleanIMContent(ctx, displaySource, tenant, s.defaultFileSvc)
display := cleanIMContent(ctx, displaySource, tenant, s.defaultFileSvc, s.storageResolver)
if err := streamer.UpdateStreamContent(ctx, msg, streamID, display); err != nil {
logger.Warnf(ctx, "[IM] UpdateStreamContent failed: %v", err)
}
@@ -2396,7 +2418,7 @@ loop:
authServices := append([]imMCPAuthService(nil), mcpAuthServices...)
bufMu.Unlock()
finalDisplay := cleanIMContent(ctx, FormatIMFinalFromParts(parts), tenant, s.defaultFileSvc)
finalDisplay := cleanIMContent(ctx, FormatIMFinalFromParts(parts), tenant, s.defaultFileSvc, s.storageResolver)
if noVisibleContent || finalDisplay == "" {
fallback := "抱歉,我暂时无法回答这个问题。"
if finalErr != nil {
@@ -2443,7 +2465,7 @@ func (s *Service) fallbackNonStream(ctx context.Context, msg *IncomingMessage, s
answer = "抱歉,处理您的问题时出现了异常,请稍后再试。"
}
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: formatIMOutboundAnswer(ctx, answer, tenant, s.defaultFileSvc), IsFinal: true})
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: formatIMOutboundAnswer(ctx, answer, tenant, s.defaultFileSvc, s.storageResolver), IsFinal: true})
}
// runQA executes the WeKnora QA pipeline and returns the full answer text.
+7
View File
@@ -297,6 +297,13 @@ func apiKeyManageVectorStores(base middleware.APIKeyRoutePolicy) middleware.APIK
return base.WithCapability(types.APIKeyCapabilityManageVectorStores)
}
// apiKeyManageStorageBackends layers the "manage_storage_backends" capability
// on top of a base policy so a scoped key can manage object/file storage
// backend instances without carrying vector-store or full tenant access.
func apiKeyManageStorageBackends(base middleware.APIKeyRoutePolicy) middleware.APIKeyRoutePolicy {
return base.WithCapability(types.APIKeyCapabilityManageStorageBackends)
}
func apiKeyManageWebSearch(base middleware.APIKeyRoutePolicy) middleware.APIKeyRoutePolicy {
return base.WithCapability(types.APIKeyCapabilityManageWebSearch)
}
+93 -22
View File
@@ -74,6 +74,8 @@ type RouterParams struct {
WebSearchProviderHandler *handler.WebSearchProviderHandler
WebSearchCredentialsHandler *handler.WebSearchProviderCredentialsHandler
VectorStoreHandler *handler.VectorStoreHandler
StorageBackendHandler *handler.StorageBackendHandler
StorageBackendResolver interfaces.StorageBackendResolver
FAQHandler *handler.FAQHandler
TagHandler *handler.TagHandler
CustomAgentHandler *handler.CustomAgentHandler
@@ -156,19 +158,19 @@ func NewRouter(params RouterParams) *gin.Engine {
RegisterIMRoutes(r, params.IMHandler)
// Web embed 公开路由(使用 publish token 鉴权,不走全局 Auth)
RegisterEmbedPublicRoutes(r, params.EmbedChannelHandler, params.EmbedChannelService, params.TenantService, params.RedisClient, params.FileService)
RegisterEmbedPublicRoutes(r, params.EmbedChannelHandler, params.EmbedChannelService, params.TenantService, params.RedisClient, params.FileService, params.StorageBackendResolver)
// 认证中间件
r.Use(middleware.Auth(params.TenantService, params.UserService, params.TenantMemberService, params.TenantAPIKeyService, params.Config))
// 文件服务:统一代理本地/MinIO/COS/TOS存储后端(需要认证)
serveFiles(r, params.FileService)
serveFiles(r, params.FileService, params.StorageBackendResolver)
// Presigned file access: no auth required, signature-verified.
servePresignedFiles(r, params.TenantService)
servePresignedFiles(r, params.TenantService, params.StorageBackendResolver)
// Diagnostic preview of presigned URLs (Admin only, behind auth middleware).
servePresignedPreview(r, params.Config)
servePresignedPreview(r, params.Config, params.StorageBackendResolver)
// Langfuse observability — only active when LANGFUSE_* env vars are set.
// The middleware is registered unconditionally; when disabled it's a no-op.
@@ -217,7 +219,7 @@ func NewRouter(params RouterParams) *gin.Engine {
// KB-scoped image proxy: lets tenants render images embedded in
// org-shared / agent-visible KB content, which the tenant-scoped
// /files route cannot serve because it enforces same-tenant paths.
serveKBScopedFiles(v1, rbacGuards, params.TenantService, params.FileService)
serveKBScopedFiles(v1, rbacGuards, params.TenantService, params.FileService, params.StorageBackendResolver)
RegisterKnowledgeTagRoutes(v1, params.TagHandler, rbacGuards)
RegisterKnowledgeRoutes(v1, params.KnowledgeHandler, rbacGuards)
RegisterFAQRoutes(v1, params.FAQHandler, rbacGuards)
@@ -234,6 +236,7 @@ func NewRouter(params RouterParams) *gin.Engine {
RegisterWebSearchRoutes(v1, params.WebSearchHandler, rbacGuards)
RegisterWebSearchProviderRoutes(v1, params.WebSearchProviderHandler, params.WebSearchCredentialsHandler, rbacGuards)
RegisterVectorStoreRoutes(v1, params.VectorStoreHandler, rbacGuards)
RegisterStorageBackendRoutes(v1, params.StorageBackendHandler, rbacGuards)
RegisterCustomAgentRoutes(v1, params.CustomAgentHandler, rbacGuards)
RegisterUserFavoriteRoutes(v1, params.UserFavoriteHandler, rbacGuards)
RegisterSkillRoutes(v1, params.SkillHandler, rbacGuards)
@@ -1075,6 +1078,22 @@ func RegisterVectorStoreRoutes(r *gin.RouterGroup, h *handler.VectorStoreHandler
}
}
// RegisterStorageBackendRoutes manages concrete object/file storage instances.
func RegisterStorageBackendRoutes(r *gin.RouterGroup, h *handler.StorageBackendHandler, g *rbacGuards) {
backends := g.apiKeyGroup(r.Group("/storage-backends"), apiKeyManageStorageBackends(apiKeyFullAccess()))
{
backends.GET("/types", g.Viewer(), h.Types)
backends.POST("/test", g.Admin(), h.TestRaw)
backends.POST("", g.Admin(), h.Create)
backends.GET("", g.Viewer(), h.List)
backends.GET("/:id", g.Viewer(), h.Get)
backends.PUT("/:id", g.Admin(), h.Update)
backends.DELETE("/:id", g.Admin(), h.Delete)
backends.POST("/:id/test", g.Admin(), h.TestByID)
backends.PUT("/:id/default", g.Admin(), h.SetDefault)
}
}
// RegisterCustomAgentRoutes registers custom agent routes.
//
// Mutating routes use OwnedAgentOrAdmin: the original creator can edit
@@ -1275,6 +1294,7 @@ func RegisterEmbedPublicRoutes(
tenantService interfaces.TenantService,
redisClient *redis.Client,
fileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
) {
if embedHandler == nil || embedService == nil {
return
@@ -1302,7 +1322,7 @@ func RegisterEmbedPublicRoutes(
// Serve images embedded in bot replies (e.g. chart exports). EmbedAuth
// injects the channel's tenant, and the handler enforces that the
// requested path belongs to that tenant.
embed.GET("/files", newFileServeHandler(fileService))
embed.GET("/files", newFileServeHandler(fileService, storageResolver))
}
}
@@ -1522,7 +1542,7 @@ type getRouteRegistrar interface {
// same handler backs both the authenticated /files route and the embed route
// (where EmbedAuth injects the channel's tenant). Tenant ownership of the
// requested path is enforced via ValidateStoragePathTenant either way.
func newFileServeHandler(globalFileService interfaces.FileService) gin.HandlerFunc {
func newFileServeHandler(globalFileService interfaces.FileService, storageResolver interfaces.StorageBackendResolver) gin.HandlerFunc {
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
@@ -1545,7 +1565,12 @@ func newFileServeHandler(globalFileService interfaces.FileService) gin.HandlerFu
return
}
provider := types.ParseProviderScheme(filePath)
backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath)
providerPath := filePath
if scoped {
providerPath = innerPath
}
provider := types.ParseProviderScheme(providerPath)
tenant, _ := c.Request.Context().Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
@@ -1565,7 +1590,9 @@ func newFileServeHandler(globalFileService interfaces.FileService) gin.HandlerFu
err error
)
if tenant.StorageEngineConfig != nil {
if storageResolver != nil {
fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(c.Request.Context(), tenant, backendID, provider, absDir)
} else if tenant.StorageEngineConfig != nil {
fileSvc, resolvedProvider, err = filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
} else {
err = http.ErrMissingFile
@@ -1608,12 +1635,16 @@ func newFileServeHandler(globalFileService interfaces.FileService) gin.HandlerFu
}
}
func serveFiles(r getRouteRegistrar, globalFileService interfaces.FileService) {
func serveFiles(r getRouteRegistrar, globalFileService interfaces.FileService, resolvers ...interfaces.StorageBackendResolver) {
logger.Infof(context.Background(), "[Router] Serving files from /files")
var storageResolver interfaces.StorageBackendResolver
if len(resolvers) > 0 {
storageResolver = resolvers[0]
}
// /files sits outside the /api/v1 APIKeyGate; storage paths cannot be
// attributed to a key's KB allow-list, so API keys are denied outright
// (embed routes use their own /embed/.../files handler).
r.GET("/files", middleware.DenyAPIKeyPrincipal(), newFileServeHandler(globalFileService))
r.GET("/files", middleware.DenyAPIKeyPrincipal(), newFileServeHandler(globalFileService, storageResolver))
}
// serveKBScopedFiles registers the KB-scoped file proxy used to render images
@@ -1633,6 +1664,7 @@ func serveKBScopedFiles(
g *rbacGuards,
tenantService interfaces.TenantService,
globalFileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
) {
logger.Infof(context.Background(), "[Router] Serving KB-scoped files from /knowledge-bases/:id/files")
// API keys are denied outright (as on /files): a signed URL's tenant/KB
@@ -1642,7 +1674,7 @@ func serveKBScopedFiles(
middleware.DenyAPIKeyPrincipal(),
g.Viewer(),
g.KBAccessRead("id"),
newKBScopedFileServeHandler(tenantService, globalFileService),
newKBScopedFileServeHandler(tenantService, globalFileService, storageResolver),
)
}
@@ -1655,7 +1687,12 @@ func serveKBScopedFiles(
func newKBScopedFileServeHandler(
tenantService interfaces.TenantService,
globalFileService interfaces.FileService,
resolvers ...interfaces.StorageBackendResolver,
) gin.HandlerFunc {
var storageResolver interfaces.StorageBackendResolver
if len(resolvers) > 0 {
storageResolver = resolvers[0]
}
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
@@ -1700,13 +1737,20 @@ func newKBScopedFileServeHandler(
return
}
provider := types.ParseProviderScheme(filePath)
backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath)
providerPath := filePath
if scoped {
providerPath = innerPath
}
provider := types.ParseProviderScheme(providerPath)
var (
fileSvc interfaces.FileService
resolvedProvider string
)
if tenant.StorageEngineConfig != nil {
if storageResolver != nil {
fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(ctx, tenant, backendID, provider, absDir)
} else if tenant.StorageEngineConfig != nil {
fileSvc, resolvedProvider, err = filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
} else {
err = http.ErrMissingFile
@@ -1766,14 +1810,14 @@ func newKBScopedFileServeHandler(
// Without this it is otherwise impossible to tell whether a "broken image" is
// caused by an expired signature, a stale URL cached by the platform, the
// platform's IP being blocked, or the URL simply never reaching us.
func servePresignedFiles(r *gin.Engine, tenantService interfaces.TenantService) {
func servePresignedFiles(r *gin.Engine, tenantService interfaces.TenantService, storageResolver interfaces.StorageBackendResolver) {
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
}
absDir, _ := filepath.Abs(baseDir)
handler := presignedFileHandler(tenantService, absDir)
handler := presignedFileHandler(tenantService, absDir, storageResolver)
r.GET("/api/v1/files/presigned", handler)
r.HEAD("/api/v1/files/presigned", handler)
}
@@ -1782,7 +1826,11 @@ func servePresignedFiles(r *gin.Engine, tenantService interfaces.TenantService)
// For HEAD requests it returns the same status + headers but does not stream
// the body — this is enough for IM platforms to validate the URL while saving
// us a full read of the backing object.
func presignedFileHandler(tenantService interfaces.TenantService, absDir string) gin.HandlerFunc {
func presignedFileHandler(tenantService interfaces.TenantService, absDir string, resolvers ...interfaces.StorageBackendResolver) gin.HandlerFunc {
var storageResolver interfaces.StorageBackendResolver
if len(resolvers) > 0 {
storageResolver = resolvers[0]
}
return func(c *gin.Context) {
ctx := c.Request.Context()
clientIP := c.ClientIP()
@@ -1823,7 +1871,12 @@ func presignedFileHandler(tenantService interfaces.TenantService, absDir string)
return
}
provider := types.ParseProviderScheme(filePath)
backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath)
providerPath := filePath
if scoped {
providerPath = innerPath
}
provider := types.ParseProviderScheme(providerPath)
tenant, err := tenantService.GetTenantByID(ctx, tenantID)
if err != nil {
logger.Warnf(ctx, "[Router] /files/presigned tenant lookup failed: client_ip=%s tenant_id=%d err=%v", clientIP, tenantID, err)
@@ -1831,7 +1884,13 @@ func presignedFileHandler(tenantService interfaces.TenantService, absDir string)
return
}
fileSvc, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
var fileSvc interfaces.FileService
var resolvedProvider string
if storageResolver != nil {
fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(ctx, tenant, backendID, provider, absDir)
} else {
fileSvc, resolvedProvider, err = filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
}
if err != nil {
logger.Warnf(ctx, "[Router] /files/presigned resolve file service failed: client_ip=%s tenant_id=%d provider=%s err=%v",
clientIP, tenantID, provider, err)
@@ -1881,7 +1940,7 @@ func presignedFileHandler(tenantService interfaces.TenantService, absDir string)
//
// Route:
// - GET /api/v1/files/presigned-preview?file_path=<provider://...>
func servePresignedPreview(r *gin.Engine, cfg *config.Config) {
func servePresignedPreview(r *gin.Engine, cfg *config.Config, storageResolver interfaces.StorageBackendResolver) {
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
@@ -1913,8 +1972,20 @@ func servePresignedPreview(r *gin.Engine, cfg *config.Config) {
return
}
provider := types.ParseProviderScheme(filePath)
fileSvc, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath)
providerPath := filePath
if scoped {
providerPath = innerPath
}
provider := types.ParseProviderScheme(providerPath)
var fileSvc interfaces.FileService
var resolvedProvider string
var err error
if storageResolver != nil {
fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(ctx, tenant, backendID, provider, absDir)
} else {
fileSvc, resolvedProvider, err = filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
}
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{
"error": err.Error(),
@@ -273,6 +273,7 @@ func TestTenantInfrastructureRoutesDeclareSpecificCapabilities(t *testing.T) {
RegisterMCPServiceRoutes(v1, &handler.MCPServiceHandler{}, &handler.MCPCredentialsHandler{}, &handler.MCPOAuthHandler{}, g)
RegisterWebSearchProviderRoutes(v1, &handler.WebSearchProviderHandler{}, &handler.WebSearchProviderCredentialsHandler{}, g)
RegisterVectorStoreRoutes(v1, &handler.VectorStoreHandler{}, g)
RegisterStorageBackendRoutes(v1, &handler.StorageBackendHandler{}, g)
RegisterEmbedChannelRoutes(v1, &handler.EmbedChannelHandler{}, g)
RegisterIMChannelRoutes(v1, &handler.IMHandler{}, g)
RegisterDataSourceRoutes(v1, &handler.DataSourceHandler{}, &handler.DataSourceCredentialsHandler{}, g)
@@ -290,6 +291,7 @@ func TestTenantInfrastructureRoutesDeclareSpecificCapabilities(t *testing.T) {
{http.MethodGet, "/api/v1/mcp-services", types.APIKeyCapabilityManageMCPServices},
{http.MethodGet, "/api/v1/web-search-providers", types.APIKeyCapabilityManageWebSearch},
{http.MethodGet, "/api/v1/vector-stores", types.APIKeyCapabilityManageVectorStores},
{http.MethodGet, "/api/v1/storage-backends", types.APIKeyCapabilityManageStorageBackends},
{http.MethodGet, "/api/v1/embed-channels", types.APIKeyCapabilityManageChannels},
{http.MethodGet, "/api/v1/im-channels", types.APIKeyCapabilityManageChannels},
{http.MethodGet, "/api/v1/datasource", types.APIKeyCapabilityManageDataSources},
@@ -0,0 +1,31 @@
package interfaces
import (
"context"
"github.com/Tencent/WeKnora/internal/types"
)
type StorageBackendRepository interface {
Create(ctx context.Context, backend *types.StorageBackend) error
GetByID(ctx context.Context, tenantID uint64, id string) (*types.StorageBackend, error)
List(ctx context.Context, tenantID uint64) ([]*types.StorageBackend, error)
Update(ctx context.Context, backend *types.StorageBackend) error
Delete(ctx context.Context, tenantID uint64, id string) error
FindLegacyAlias(ctx context.Context, tenantID uint64, provider string) (*types.StorageBackend, error)
}
type StorageBackendService interface {
Create(ctx context.Context, backend *types.StorageBackend) error
Update(ctx context.Context, backend *types.StorageBackend) error
Delete(ctx context.Context, tenantID uint64, id string) error
SetDefault(ctx context.Context, tenantID uint64, id string) error
Test(ctx context.Context, backend *types.StorageBackend) error
}
// StorageBackendResolver is the single runtime entry point for resolving one
// concrete storage instance. backendID wins; provider is a legacy fallback.
type StorageBackendResolver interface {
ResolveFileService(ctx context.Context, tenant *types.Tenant, backendID, provider, localBaseDir string) (FileService, string, error)
ResolveBackend(ctx context.Context, tenant *types.Tenant, backendID, provider string) (*types.StorageBackend, error)
}
+45
View File
@@ -9,6 +9,24 @@ import (
"gorm.io/gorm"
)
const storageBackendScheme = "storage://"
func BuildStorageBackendPath(backendID, providerPath string) string {
return storageBackendScheme + strings.TrimSpace(backendID) + "/" + providerPath
}
func ParseStorageBackendPath(path string) (backendID, providerPath string, ok bool) {
if !strings.HasPrefix(path, storageBackendScheme) {
return "", "", false
}
rest := strings.TrimPrefix(path, storageBackendScheme)
parts := strings.SplitN(rest, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return "", "", false
}
return parts[0], parts[1], true
}
// KnowledgeBaseType represents the type of the knowledge base
const (
// KnowledgeBaseTypeDocument represents the document knowledge base type
@@ -71,6 +89,9 @@ type KnowledgeBase struct {
ASRConfig ASRConfig `yaml:"asr_config" json:"asr_config" gorm:"type:json"`
// Storage provider config (new): only stores provider selection; credentials from workspace StorageEngineConfig
StorageProviderConfig *StorageProviderConfig `yaml:"storage_provider_config" json:"storage_provider_config" gorm:"column:storage_provider_config;type:jsonb"`
// StorageBackendID binds this KB to one concrete storage instance. The
// legacy provider field remains readable during migration only.
StorageBackendID *string `yaml:"storage_backend_id" json:"storage_backend_id,omitempty" gorm:"column:storage_backend_id;type:varchar(36);default:null"`
// Deprecated: legacy COS config column. Kept for backward compatibility with old data.
StorageConfig StorageConfig `yaml:"-" json:"storage_config" gorm:"column:cos_config;type:json"`
// VectorStoreID references the VectorStore this knowledge base is bound to.
@@ -317,6 +338,27 @@ func (kb *KnowledgeBase) SetStorageProvider(provider string) {
kb.StorageProviderConfig = &StorageProviderConfig{Provider: provider}
}
// SharesStorageBackendWith compares concrete instance bindings first. Provider
// comparison is only a compatibility fallback for rows not yet backfilled.
func (kb *KnowledgeBase) SharesStorageBackendWith(other *KnowledgeBase, defaultBackendID, defaultProvider string) bool {
if kb == nil || other == nil {
return false
}
effectiveID := func(candidate *KnowledgeBase) string {
if candidate.StorageBackendID != nil {
if id := strings.TrimSpace(*candidate.StorageBackendID); id != "" {
return id
}
}
return strings.TrimSpace(defaultBackendID)
}
leftID, rightID := effectiveID(kb), effectiveID(other)
if leftID != "" || rightID != "" {
return leftID != "" && leftID == rightID
}
return kb.EffectiveStorageProvider(defaultProvider) == other.EffectiveStorageProvider(defaultProvider)
}
// InferStorageFromFilePath deduces the storage provider from a file path format.
// Used as a safety fallback when the KB's configured provider doesn't match the data.
// Supports provider:// scheme (local://, minio://, cos://, tos://),
@@ -339,6 +381,9 @@ func InferStorageFromFilePath(filePath string) string {
// e.g. "minio://bucket/key" → "minio", "local://tenant/file.pdf" → "local"
// Returns "" if the path does not use a known provider scheme.
func ParseProviderScheme(filePath string) string {
if _, inner, ok := ParseStorageBackendPath(filePath); ok {
filePath = inner
}
for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3", "obs"} {
if strings.HasPrefix(filePath, provider+"://") {
return provider
+357
View File
@@ -0,0 +1,357 @@
package types
import (
"database/sql/driver"
"encoding/json"
"fmt"
"os"
"path"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/storageallowlist"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"gorm.io/gorm"
)
const (
StorageBackendSourceUser = "user"
StorageBackendSourceEnv = "env"
StorageBackendStatusActive = "active"
StorageBackendStatusDisabled = "disabled"
)
// StorageBackend is one concrete file/object storage instance. A workspace may
// register multiple instances of the same provider and bind each knowledge base
// to a different instance.
type StorageBackend struct {
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
TenantID uint64 `json:"tenant_id" gorm:"not null;index"`
Name string `json:"name" gorm:"type:varchar(255);not null"`
Provider string `json:"provider" gorm:"type:varchar(32);not null;index"`
Config StorageBackendConfig `json:"config" gorm:"type:json"`
Source string `json:"source" gorm:"type:varchar(16);not null;default:'user'"`
Status string `json:"status" gorm:"type:varchar(16);not null;default:'active'"`
LegacyAlias bool `json:"legacy_alias" gorm:"not null;default:false"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
}
func (StorageBackend) TableName() string { return "storage_backends" }
func (b *StorageBackend) BeforeCreate(_ *gorm.DB) error {
if b.ID == "" {
b.ID = uuid.NewString()
}
if b.Source == "" {
b.Source = StorageBackendSourceUser
}
if b.Status == "" {
b.Status = StorageBackendStatusActive
}
return nil
}
func (b *StorageBackend) Validate() error {
if b.TenantID == 0 {
return errors.NewValidationError("tenant_id is required")
}
b.Name = strings.TrimSpace(b.Name)
if b.Name == "" {
return errors.NewValidationError("name is required")
}
b.Provider = strings.ToLower(strings.TrimSpace(b.Provider))
if !storageallowlist.IsAllowed(b.Provider) {
return errors.NewValidationError(fmt.Sprintf("storage provider %q is not allowed", b.Provider))
}
if !isSupportedStorageBackendProvider(b.Provider) {
return errors.NewValidationError(fmt.Sprintf("unsupported storage provider: %s", b.Provider))
}
if b.Status == "" {
b.Status = StorageBackendStatusActive
}
if b.Status != StorageBackendStatusActive && b.Status != StorageBackendStatusDisabled {
return errors.NewValidationError("status must be active or disabled")
}
return b.Config.ValidateForProvider(b.Provider)
}
func isSupportedStorageBackendProvider(provider string) bool {
for _, candidate := range storageallowlist.Supported() {
if provider == candidate {
return true
}
}
return false
}
// StorageBackendConfig is the normalized union of provider-specific storage
// settings. AccessKeyID/SecretAccessKey map to COS SecretID/SecretKey and to
// the access/secret key pair used by S3-compatible providers.
type StorageBackendConfig struct {
Mode string `json:"mode,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
Region string `json:"region,omitempty"`
AccessKeyID string `json:"access_key_id,omitempty"`
SecretAccessKey string `json:"secret_access_key,omitempty"`
BucketName string `json:"bucket_name,omitempty"`
PathPrefix string `json:"path_prefix,omitempty"`
AppID string `json:"app_id,omitempty"`
UseSSL bool `json:"use_ssl,omitempty"`
ForcePathStyle bool `json:"force_path_style,omitempty"`
UseTempBucket bool `json:"use_temp_bucket,omitempty"`
TempBucketName string `json:"temp_bucket_name,omitempty"`
TempRegion string `json:"temp_region,omitempty"`
}
func (c StorageBackendConfig) Value() (driver.Value, error) {
if key := utils.GetAESKey(); key != nil {
if c.AccessKeyID != "" {
if encrypted, err := utils.EncryptAESGCM(c.AccessKeyID, key); err == nil {
c.AccessKeyID = encrypted
}
}
if c.SecretAccessKey != "" {
if encrypted, err := utils.EncryptAESGCM(c.SecretAccessKey, key); err == nil {
c.SecretAccessKey = encrypted
}
}
}
return json.Marshal(c)
}
func (c *StorageBackendConfig) Scan(value interface{}) error {
if value == nil {
return nil
}
b, ok := value.([]byte)
if !ok {
return nil
}
if err := json.Unmarshal(b, c); err != nil {
return err
}
accessKey, err := utils.DecryptStoredSecret(c.AccessKeyID)
if err != nil {
return fmt.Errorf("decrypt storage backend access key: %w", err)
}
secretKey, err := utils.DecryptStoredSecret(c.SecretAccessKey)
if err != nil {
return fmt.Errorf("decrypt storage backend secret key: %w", err)
}
c.AccessKeyID = accessKey
c.SecretAccessKey = secretKey
return nil
}
func (c StorageBackendConfig) MaskSensitiveFields() StorageBackendConfig {
out := c
if out.AccessKeyID != "" {
out.AccessKeyID = RedactedSecretPlaceholder
}
if out.SecretAccessKey != "" {
out.SecretAccessKey = RedactedSecretPlaceholder
}
return out
}
func (c StorageBackendConfig) MergeSecrets(existing StorageBackendConfig) StorageBackendConfig {
c.AccessKeyID = PreserveIfRedacted(c.AccessKeyID, existing.AccessKeyID)
c.SecretAccessKey = PreserveIfRedacted(c.SecretAccessKey, existing.SecretAccessKey)
return c
}
func (c StorageBackendConfig) ValidateForProvider(provider string) error {
prefix := strings.ReplaceAll(strings.TrimSpace(c.PathPrefix), "\\", "/")
cleanPrefix := path.Clean(prefix)
if strings.HasPrefix(prefix, "/") || cleanPrefix == ".." || strings.HasPrefix(cleanPrefix, "../") {
return errors.NewValidationError("path_prefix must be a relative path without parent traversal")
}
required := func(name, value string) error {
if strings.TrimSpace(value) == "" {
return errors.NewValidationError(name + " is required")
}
return nil
}
switch provider {
case "local":
return nil
case "minio":
if c.Mode == "" {
c.Mode = "remote"
}
if c.Mode != "docker" {
for name, value := range map[string]string{"endpoint": c.Endpoint, "access_key_id": c.AccessKeyID, "secret_access_key": c.SecretAccessKey} {
if err := required(name, value); err != nil {
return err
}
}
}
return required("bucket_name", c.BucketName)
case "cos":
for name, value := range map[string]string{"region": c.Region, "access_key_id": c.AccessKeyID, "secret_access_key": c.SecretAccessKey, "bucket_name": c.BucketName} {
if err := required(name, value); err != nil {
return err
}
}
return nil
default:
for name, value := range map[string]string{"endpoint": c.Endpoint, "region": c.Region, "access_key_id": c.AccessKeyID, "secret_access_key": c.SecretAccessKey, "bucket_name": c.BucketName} {
if err := required(name, value); err != nil {
return err
}
}
return nil
}
}
// LocationKey identifies the physical destination. Credentials deliberately do
// not participate so they can be rotated without changing object identity.
func (c StorageBackendConfig) LocationKey(provider string) string {
mode := strings.TrimSpace(c.Mode)
if provider == "minio" && mode == "" {
mode = "remote"
}
return strings.Join([]string{provider, mode, strings.TrimSpace(c.Endpoint), strings.TrimSpace(c.Region), strings.TrimSpace(c.BucketName), strings.Trim(strings.TrimSpace(c.PathPrefix), "/")}, "|")
}
// ToStorageEngineConfig adapts the instance model to the existing provider
// implementations while those implementations are progressively normalized.
func (b StorageBackend) ToStorageEngineConfig() *StorageEngineConfig {
c := b.Config
result := &StorageEngineConfig{DefaultProvider: b.Provider}
switch b.Provider {
case "local":
result.Local = &LocalEngineConfig{PathPrefix: c.PathPrefix}
case "minio":
mode := c.Mode
if mode == "" {
mode = "remote"
}
result.MinIO = &MinIOEngineConfig{Mode: mode, Endpoint: c.Endpoint, AccessKeyID: c.AccessKeyID, SecretAccessKey: c.SecretAccessKey, BucketName: c.BucketName, UseSSL: c.UseSSL, PathPrefix: c.PathPrefix}
case "cos":
result.COS = &COSEngineConfig{SecretID: c.AccessKeyID, SecretKey: c.SecretAccessKey, Region: c.Region, BucketName: c.BucketName, AppID: c.AppID, PathPrefix: c.PathPrefix, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
case "tos":
result.TOS = &TOSEngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
case "s3":
result.S3 = &S3EngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseSSL: c.UseSSL, ForcePathStyle: c.ForcePathStyle}
case "oss":
result.OSS = &OSSEngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseTempBucket: c.UseTempBucket, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
case "ks3":
result.KS3 = &KS3EngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix}
case "obs":
result.OBS = &OBSEngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseSSL: c.UseSSL}
}
return result
}
func NewStorageBackendResponse(backend *StorageBackend) StorageBackend {
out := *backend
out.Config = backend.Config.MaskSensitiveFields()
return out
}
// StorageBackendFromLegacy projects one provider entry from the old workspace
// singleton JSON into the multi-instance model.
func StorageBackendFromLegacy(tenantID uint64, provider string, legacy *StorageEngineConfig) *StorageBackend {
if legacy == nil {
return nil
}
provider = strings.ToLower(strings.TrimSpace(provider))
b := &StorageBackend{TenantID: tenantID, Provider: provider, Source: StorageBackendSourceUser, Status: StorageBackendStatusActive, LegacyAlias: true}
switch provider {
case "local":
if legacy.Local == nil {
return nil
}
b.Name, b.Config.PathPrefix = "Local", legacy.Local.PathPrefix
case "minio":
if legacy.MinIO == nil {
return nil
}
c := legacy.MinIO
b.Name = "MinIO"
b.Config = StorageBackendConfig{Mode: c.Mode, Endpoint: c.Endpoint, AccessKeyID: c.AccessKeyID, SecretAccessKey: c.SecretAccessKey, BucketName: c.BucketName, UseSSL: c.UseSSL, PathPrefix: c.PathPrefix}
case "cos":
if legacy.COS == nil {
return nil
}
c := legacy.COS
b.Name = "COS"
b.Config = StorageBackendConfig{Region: c.Region, AccessKeyID: c.SecretID, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, AppID: c.AppID, PathPrefix: c.PathPrefix, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
case "tos":
if legacy.TOS == nil {
return nil
}
c := legacy.TOS
b.Name = "TOS"
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
case "s3":
if legacy.S3 == nil {
return nil
}
c := legacy.S3
b.Name = "S3"
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseSSL: c.UseSSL, ForcePathStyle: c.ForcePathStyle}
case "oss":
if legacy.OSS == nil {
return nil
}
c := legacy.OSS
b.Name = "OSS"
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseTempBucket: c.UseTempBucket, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
case "ks3":
if legacy.KS3 == nil {
return nil
}
c := legacy.KS3
b.Name = "KS3"
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix}
case "obs":
if legacy.OBS == nil {
return nil
}
c := legacy.OBS
b.Name = "OBS"
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseSSL: c.UseSSL}
default:
return nil
}
return b
}
// StorageBackendFromEnvironment snapshots the process-wide storage backend for
// a workspace. The row is read-only in the UI and keeps env-only deployments
// on the same instance-resolution path as user-managed backends.
func StorageBackendFromEnvironment(tenantID uint64) *StorageBackend {
provider := strings.ToLower(strings.TrimSpace(os.Getenv("STORAGE_TYPE")))
if provider == "" {
provider = "local"
}
b := &StorageBackend{
TenantID: tenantID, Name: "System " + strings.ToUpper(provider), Provider: provider,
Source: StorageBackendSourceEnv, Status: StorageBackendStatusActive, LegacyAlias: true,
}
switch provider {
case "local":
b.Config.PathPrefix = strings.TrimSpace(os.Getenv("LOCAL_STORAGE_PATH_PREFIX"))
case "minio":
b.Config = StorageBackendConfig{Mode: "remote", Endpoint: os.Getenv("MINIO_ENDPOINT"), AccessKeyID: os.Getenv("MINIO_ACCESS_KEY_ID"), SecretAccessKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"), BucketName: os.Getenv("MINIO_BUCKET_NAME"), PathPrefix: os.Getenv("MINIO_PATH_PREFIX"), UseSSL: strings.EqualFold(os.Getenv("MINIO_USE_SSL"), "true")}
case "cos":
b.Config = StorageBackendConfig{Region: os.Getenv("COS_REGION"), AccessKeyID: os.Getenv("COS_SECRET_ID"), SecretAccessKey: os.Getenv("COS_SECRET_KEY"), BucketName: os.Getenv("COS_BUCKET_NAME"), AppID: os.Getenv("COS_APP_ID"), PathPrefix: os.Getenv("COS_PATH_PREFIX"), TempBucketName: os.Getenv("COS_TEMP_BUCKET_NAME"), TempRegion: os.Getenv("COS_TEMP_REGION")}
case "tos":
b.Config = StorageBackendConfig{Endpoint: os.Getenv("TOS_ENDPOINT"), Region: os.Getenv("TOS_REGION"), AccessKeyID: os.Getenv("TOS_ACCESS_KEY"), SecretAccessKey: os.Getenv("TOS_SECRET_KEY"), BucketName: os.Getenv("TOS_BUCKET_NAME"), PathPrefix: os.Getenv("TOS_PATH_PREFIX"), TempBucketName: os.Getenv("TOS_TEMP_BUCKET_NAME"), TempRegion: os.Getenv("TOS_TEMP_REGION")}
case "s3":
b.Config = StorageBackendConfig{Endpoint: os.Getenv("S3_ENDPOINT"), Region: os.Getenv("S3_REGION"), AccessKeyID: os.Getenv("S3_ACCESS_KEY"), SecretAccessKey: os.Getenv("S3_SECRET_KEY"), BucketName: os.Getenv("S3_BUCKET_NAME"), PathPrefix: os.Getenv("S3_PATH_PREFIX"), UseSSL: !strings.EqualFold(os.Getenv("S3_USE_SSL"), "false"), ForcePathStyle: strings.EqualFold(os.Getenv("S3_FORCE_PATH_STYLE"), "true")}
case "oss":
b.Config = StorageBackendConfig{Endpoint: os.Getenv("OSS_ENDPOINT"), Region: os.Getenv("OSS_REGION"), AccessKeyID: os.Getenv("OSS_ACCESS_KEY"), SecretAccessKey: os.Getenv("OSS_SECRET_KEY"), BucketName: os.Getenv("OSS_BUCKET_NAME"), PathPrefix: os.Getenv("OSS_PATH_PREFIX"), UseTempBucket: os.Getenv("OSS_TEMP_BUCKET_NAME") != "", TempBucketName: os.Getenv("OSS_TEMP_BUCKET_NAME"), TempRegion: os.Getenv("OSS_TEMP_REGION")}
case "obs":
b.Config = StorageBackendConfig{Endpoint: os.Getenv("OBS_ENDPOINT"), Region: os.Getenv("OBS_REGION"), AccessKeyID: os.Getenv("OBS_ACCESS_KEY"), SecretAccessKey: os.Getenv("OBS_SECRET_KEY"), BucketName: os.Getenv("OBS_BUCKET_NAME"), PathPrefix: os.Getenv("OBS_PATH_PREFIX"), UseSSL: !strings.EqualFold(os.Getenv("OBS_USE_SSL"), "false")}
default:
return nil
}
return b
}
+60
View File
@@ -0,0 +1,60 @@
package types
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestStorageBackendPathRoundTrip(t *testing.T) {
path := BuildStorageBackendPath("backend-a", "cos://bucket/region/7/file.pdf")
id, inner, ok := ParseStorageBackendPath(path)
require.True(t, ok)
assert.Equal(t, "backend-a", id)
assert.Equal(t, "cos://bucket/region/7/file.pdf", inner)
assert.Equal(t, "cos", ParseProviderScheme(path))
}
func TestSharesStorageBackendWithUsesConcreteInstance(t *testing.T) {
aID, bID := "cos-a", "cos-b"
a := &KnowledgeBase{StorageBackendID: &aID, StorageProviderConfig: &StorageProviderConfig{Provider: "cos"}}
b := &KnowledgeBase{StorageBackendID: &bID, StorageProviderConfig: &StorageProviderConfig{Provider: "cos"}}
assert.False(t, a.SharesStorageBackendWith(b, "", "cos"))
b.StorageBackendID = &aID
assert.True(t, a.SharesStorageBackendWith(b, "", "cos"))
}
func TestNewStorageBackendResponseMasksCredentials(t *testing.T) {
backend := &StorageBackend{Config: StorageBackendConfig{AccessKeyID: "id", SecretAccessKey: "secret"}}
response := NewStorageBackendResponse(backend)
assert.Equal(t, RedactedSecretPlaceholder, response.Config.AccessKeyID)
assert.Equal(t, RedactedSecretPlaceholder, response.Config.SecretAccessKey)
assert.Equal(t, "id", backend.Config.AccessKeyID)
}
func TestStorageBackendFromEnvironment(t *testing.T) {
t.Setenv("STORAGE_TYPE", "s3")
t.Setenv("S3_ENDPOINT", "https://s3.example.com")
t.Setenv("S3_REGION", "ap-test-1")
t.Setenv("S3_ACCESS_KEY", "access")
t.Setenv("S3_SECRET_KEY", "secret")
t.Setenv("S3_BUCKET_NAME", "bucket")
backend := StorageBackendFromEnvironment(42)
require.NotNil(t, backend)
assert.Equal(t, uint64(42), backend.TenantID)
assert.Equal(t, "s3", backend.Provider)
assert.Equal(t, StorageBackendSourceEnv, backend.Source)
assert.True(t, backend.LegacyAlias)
assert.Equal(t, "bucket", backend.Config.BucketName)
}
func TestStorageBackendRejectsTraversingPathPrefix(t *testing.T) {
backend := &StorageBackend{
TenantID: 1, Name: "unsafe", Provider: "local",
Config: StorageBackendConfig{PathPrefix: "../outside"},
}
require.Error(t, backend.Validate())
}
+18 -12
View File
@@ -110,6 +110,8 @@ type Tenant struct {
Credentials *CredentialsConfig `yaml:"credentials" json:"credentials" gorm:"type:jsonb"`
// Storage engine config: parameters for Local, MinIO, COS. Used for document/file storage and docreader.
StorageEngineConfig *StorageEngineConfig `yaml:"storage_engine_config" json:"storage_engine_config" gorm:"type:jsonb"`
// DefaultStorageBackendID is the workspace default concrete storage instance.
DefaultStorageBackendID *string `yaml:"default_storage_backend_id" json:"default_storage_backend_id,omitempty" gorm:"column:default_storage_backend_id;type:varchar(36)"`
// Chat history config: knowledge base configuration for indexing and searching chat messages via vector search
ChatHistoryConfig *ChatHistoryConfig `yaml:"chat_history_config" json:"chat_history_config" gorm:"type:jsonb"`
// Retrieval config: global search/retrieval parameters shared by knowledge search and message search
@@ -478,22 +480,26 @@ type MinIOEngineConfig struct {
// COSEngineConfig is for Tencent Cloud COS.
type COSEngineConfig struct {
SecretID string `json:"secret_id"`
SecretKey string `json:"secret_key"`
Region string `json:"region"`
BucketName string `json:"bucket_name"`
AppID string `json:"app_id"`
PathPrefix string `json:"path_prefix"`
SecretID string `json:"secret_id"`
SecretKey string `json:"secret_key"`
Region string `json:"region"`
BucketName string `json:"bucket_name"`
AppID string `json:"app_id"`
PathPrefix string `json:"path_prefix"`
TempBucketName string `json:"temp_bucket_name"`
TempRegion string `json:"temp_region"`
}
// TOSEngineConfig is for Volcengine TOS (火山引擎对象存储).
type TOSEngineConfig struct {
Endpoint string `json:"endpoint"`
Region string `json:"region"`
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
BucketName string `json:"bucket_name"`
PathPrefix string `json:"path_prefix"`
Endpoint string `json:"endpoint"`
Region string `json:"region"`
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
BucketName string `json:"bucket_name"`
PathPrefix string `json:"path_prefix"`
TempBucketName string `json:"temp_bucket_name"`
TempRegion string `json:"temp_region"`
}
// S3EngineConfig is for AWS S3 and S3-compatible object storage.
+10
View File
@@ -97,6 +97,14 @@ const (
// APIKeyCapabilityManageVectorStores lets a key manage retrieval
// infrastructure such as vector stores, parser engines, and storage checks.
APIKeyCapabilityManageVectorStores APIKeyCapability = "manage_vector_stores"
// APIKeyCapabilityManageStorageBackends lets a key manage object/file
// storage backend instances (e.g. S3-compatible or local file storage):
// their CRUD lifecycle, connectivity tests, and the tenant default
// selection. It is separate from manage_vector_stores because storage
// backends are the file-persistence layer (holding object-store
// credentials and user-controllable endpoints) rather than retrieval
// infrastructure.
APIKeyCapabilityManageStorageBackends APIKeyCapability = "manage_storage_backends"
// APIKeyCapabilityManageWebSearch lets a key manage tenant web-search
// provider configurations and credentials.
APIKeyCapabilityManageWebSearch APIKeyCapability = "manage_web_search"
@@ -148,6 +156,8 @@ func NormalizeAPIKeyCapability(c APIKeyCapability) APIKeyCapability {
return APIKeyCapabilityManageChannels
case APIKeyCapabilityManageVectorStores:
return APIKeyCapabilityManageVectorStores
case APIKeyCapabilityManageStorageBackends:
return APIKeyCapabilityManageStorageBackends
case APIKeyCapabilityManageWebSearch:
return APIKeyCapabilityManageWebSearch
case APIKeyCapabilityRunEvaluations:
+1
View File
@@ -21,5 +21,6 @@ DROP TABLE IF EXISTS messages;
DROP TABLE IF EXISTS sessions;
DROP TABLE IF EXISTS knowledges;
DROP TABLE IF EXISTS knowledge_bases;
DROP TABLE IF EXISTS storage_backends;
DROP TABLE IF EXISTS models;
DROP TABLE IF EXISTS tenants;
+23
View File
@@ -15,6 +15,7 @@ CREATE TABLE IF NOT EXISTS tenants (
web_search_config TEXT DEFAULT NULL,
parser_engine_config TEXT DEFAULT NULL,
storage_engine_config TEXT DEFAULT NULL,
default_storage_backend_id VARCHAR(36),
credentials TEXT DEFAULT NULL,
chat_history_config TEXT,
retrieval_config TEXT,
@@ -69,6 +70,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
pinned_at DATETIME NULL,
asr_config TEXT,
vector_store_id VARCHAR(36),
storage_backend_id VARCHAR(36),
creator_id VARCHAR(36),
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
@@ -78,6 +80,8 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_id ON knowledge_bases(tenant_id);
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_vector_store
ON knowledge_bases(tenant_id, vector_store_id);
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_storage_backend
ON knowledge_bases(tenant_id, storage_backend_id);
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_creator
ON knowledge_bases(tenant_id, creator_id);
@@ -804,6 +808,25 @@ CREATE INDEX IF NOT EXISTS idx_vector_stores_tenant_id ON vector_stores(tenant_i
CREATE INDEX IF NOT EXISTS idx_vector_stores_engine_type ON vector_stores(engine_type);
CREATE INDEX IF NOT EXISTS idx_vector_stores_deleted_at ON vector_stores(deleted_at);
CREATE TABLE IF NOT EXISTS storage_backends (
id VARCHAR(36) PRIMARY KEY,
tenant_id INTEGER NOT NULL,
name VARCHAR(255) NOT NULL,
provider VARCHAR(32) NOT NULL,
config TEXT NOT NULL DEFAULT '{}',
source VARCHAR(16) NOT NULL DEFAULT 'user',
status VARCHAR(16) NOT NULL DEFAULT 'active',
legacy_alias BOOLEAN NOT NULL DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
deleted_at DATETIME
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_name_tenant
ON storage_backends(tenant_id, name) WHERE deleted_at IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_legacy_alias
ON storage_backends(tenant_id, provider) WHERE deleted_at IS NULL AND legacy_alias = 1;
CREATE INDEX IF NOT EXISTS idx_storage_backends_tenant ON storage_backends(tenant_id);
CREATE TABLE IF NOT EXISTS tenant_api_keys (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant_id INTEGER NOT NULL,
@@ -0,0 +1,4 @@
DROP INDEX IF EXISTS idx_knowledge_bases_storage_backend;
ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS storage_backend_id;
ALTER TABLE tenants DROP COLUMN IF EXISTS default_storage_backend_id;
DROP TABLE IF EXISTS storage_backends;
@@ -0,0 +1,19 @@
CREATE TABLE IF NOT EXISTS storage_backends (
id VARCHAR(36) NOT NULL PRIMARY KEY,
tenant_id BIGINT NOT NULL,
name VARCHAR(255) NOT NULL,
provider VARCHAR(32) NOT NULL,
config JSONB NOT NULL DEFAULT '{}',
source VARCHAR(16) NOT NULL DEFAULT 'user',
status VARCHAR(16) NOT NULL DEFAULT 'active',
legacy_alias BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_name_tenant ON storage_backends(tenant_id, name) WHERE deleted_at IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_legacy_alias ON storage_backends(tenant_id, provider) WHERE deleted_at IS NULL AND legacy_alias = TRUE;
CREATE INDEX IF NOT EXISTS idx_storage_backends_tenant ON storage_backends(tenant_id);
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS default_storage_backend_id VARCHAR(36);
ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS storage_backend_id VARCHAR(36);
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_storage_backend ON knowledge_bases(tenant_id, storage_backend_id);