feat(storage): add multi-instance storage backends per workspace
Introduce tenant-scoped storage backend registry so workspaces can register multiple storage instances and bind each knowledge base to a specific backend, with migration from legacy tenant-level storage config and new settings UI.
This commit is contained in:
@@ -142,6 +142,7 @@ export interface KBModelConfigRequest {
|
||||
enabled: boolean
|
||||
}
|
||||
/** 存储引擎选择:"local" | "minio" | "cos" | "obs" 等,影响文档上传与文档内图片存储 */
|
||||
storageBackendId?: string
|
||||
storageProvider?: string
|
||||
nodeExtract: {
|
||||
enabled: boolean
|
||||
|
||||
@@ -65,6 +65,9 @@ export function createKnowledgeBase(data: {
|
||||
// store. Immutable after creation — UpdateKnowledgeBase intentionally
|
||||
// does not accept this field.
|
||||
vector_store_id?: string;
|
||||
// Concrete tenant-owned storage instance. When omitted, the tenant default
|
||||
// backend is bound by the server at creation time.
|
||||
storage_backend_id?: string;
|
||||
vlm_config?: {
|
||||
enabled: boolean;
|
||||
model_id?: string;
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { get, post, put, del } from '@/utils/request'
|
||||
|
||||
export interface StorageBackendConfig {
|
||||
mode?: string
|
||||
endpoint?: string
|
||||
region?: string
|
||||
access_key_id?: string
|
||||
secret_access_key?: string
|
||||
bucket_name?: string
|
||||
path_prefix?: string
|
||||
app_id?: string
|
||||
use_ssl?: boolean
|
||||
force_path_style?: boolean
|
||||
use_temp_bucket?: boolean
|
||||
temp_bucket_name?: string
|
||||
temp_region?: string
|
||||
}
|
||||
|
||||
export interface StorageBackend {
|
||||
id: string
|
||||
tenant_id?: number
|
||||
name: string
|
||||
provider: string
|
||||
config: StorageBackendConfig
|
||||
source: 'user' | 'env'
|
||||
status: 'active' | 'disabled'
|
||||
legacy_alias?: boolean
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
}
|
||||
|
||||
export interface StorageBackendListResponse {
|
||||
success: boolean
|
||||
data: StorageBackend[]
|
||||
default_storage_backend_id?: string | null
|
||||
}
|
||||
|
||||
export const listStorageBackends = (): Promise<StorageBackendListResponse> => get('/api/v1/storage-backends')
|
||||
export const listStorageBackendTypes = (): Promise<{ success: boolean; data: string[] }> => get('/api/v1/storage-backends/types')
|
||||
export const createStorageBackend = (data: Partial<StorageBackend>) => post('/api/v1/storage-backends', data)
|
||||
export const updateStorageBackend = (id: string, data: Partial<StorageBackend>) => put(`/api/v1/storage-backends/${id}`, data)
|
||||
export const deleteStorageBackend = (id: string) => del(`/api/v1/storage-backends/${id}`)
|
||||
export const setDefaultStorageBackend = (id: string) => put(`/api/v1/storage-backends/${id}/default`, {})
|
||||
export const testStorageBackend = (data: Partial<StorageBackend>) => post('/api/v1/storage-backends/test', data)
|
||||
export const testStorageBackendByID = (id: string) => post(`/api/v1/storage-backends/${id}/test`, {})
|
||||
@@ -79,6 +79,7 @@ export type TenantAPIKeyCapability =
|
||||
| 'manage_datasources'
|
||||
| 'manage_channels'
|
||||
| 'manage_vector_stores'
|
||||
| 'manage_storage_backends'
|
||||
| 'manage_web_search'
|
||||
| 'run_evaluations'
|
||||
| 'manage_members'
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
<svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
|
||||
<title>Local storage</title>
|
||||
<rect x="3" y="4" width="18" height="7" rx="2" fill="#5C6678"/>
|
||||
<rect x="3" y="13" width="18" height="7" rx="2" fill="#434C5E"/>
|
||||
<rect x="5.5" y="7.1" width="13" height="0.8" rx="0.4" fill="#8A94A6"/>
|
||||
<rect x="5.5" y="16.1" width="13" height="0.8" rx="0.4" fill="#8A94A6"/>
|
||||
<circle cx="7.2" cy="7.5" r="0.9" fill="#6B7280"/>
|
||||
<circle cx="7.2" cy="16.5" r="0.9" fill="#6B7280"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 499 B |
@@ -6240,6 +6240,8 @@ export default {
|
||||
capabilityManageChannelsHint: 'Manage agent embed channels, IM channels, and WeChat QR binding flows.',
|
||||
capabilityManageVectorStores: 'Manage retrieval infrastructure',
|
||||
capabilityManageVectorStoresHint: 'Manage vector-store configuration plus parser, document reader, and storage engine connectivity checks.',
|
||||
capabilityManageStorageBackends: 'Manage storage backends',
|
||||
capabilityManageStorageBackendsHint: 'Manage object/file storage backend instances (e.g. S3-compatible or local file storage): their CRUD lifecycle, connectivity tests, and the tenant default selection.',
|
||||
capabilityManageWebSearch: 'Manage web search',
|
||||
capabilityManageWebSearchHint: 'Manage web-search provider configurations, credentials, and connection tests.',
|
||||
capabilityRunEvaluations: 'Run evaluations',
|
||||
|
||||
@@ -6244,6 +6244,8 @@ export default {
|
||||
capabilityManageChannelsHint: "에이전트 임베드 채널, IM 채널 및 WeChat QR 바인딩 흐름을 관리합니다.",
|
||||
capabilityManageVectorStores: "검색 인프라 관리",
|
||||
capabilityManageVectorStoresHint: "벡터 스토어 설정과 파서, 문서 리더, 스토리지 엔진 연결 검사를 관리합니다.",
|
||||
capabilityManageStorageBackends: "스토리지 백엔드 관리",
|
||||
capabilityManageStorageBackendsHint: "객체/파일 스토리지 백엔드 인스턴스(예: S3 호환 또는 로컬 파일 스토리지)의 CRUD, 연결 테스트 및 테넌트 기본 스토리지 설정을 관리합니다.",
|
||||
capabilityManageWebSearch: "웹 검색 관리",
|
||||
capabilityManageWebSearchHint: "웹 검색 공급자 설정, 자격 증명 및 연결 테스트를 관리합니다.",
|
||||
capabilityRunEvaluations: "평가 실행",
|
||||
|
||||
@@ -6065,6 +6065,8 @@ export default {
|
||||
capabilityManageChannelsHint: 'Управление embed-каналами агентов, IM-каналами и привязкой WeChat через QR-код.',
|
||||
capabilityManageVectorStores: 'Управление инфраструктурой поиска',
|
||||
capabilityManageVectorStoresHint: 'Управление векторными хранилищами и проверками парсеров, чтения документов и storage engine.',
|
||||
capabilityManageStorageBackends: 'Управление хранилищами (backends)',
|
||||
capabilityManageStorageBackendsHint: 'Управление экземплярами объектных/файловых хранилищ (например, S3-совместимых или локальных): их CRUD, проверки подключения и выбор хранилища по умолчанию для арендатора.',
|
||||
capabilityManageWebSearch: 'Управление веб-поиском',
|
||||
capabilityManageWebSearchHint: 'Управление провайдерами веб-поиска, учётными данными и проверками подключения.',
|
||||
capabilityRunEvaluations: 'Запуск оценок',
|
||||
|
||||
@@ -6265,6 +6265,8 @@ export default {
|
||||
capabilityManageChannelsHint: "允许管理智能体的嵌入渠道、IM 渠道和微信扫码绑定流程。",
|
||||
capabilityManageVectorStores: "管理检索基础设施",
|
||||
capabilityManageVectorStoresHint: "允许管理向量库配置,以及解析器、文档读取器和存储引擎连通性检查。",
|
||||
capabilityManageStorageBackends: "管理存储后端",
|
||||
capabilityManageStorageBackendsHint: "允许管理对象/文件存储后端实例(如 S3 兼容或本地文件存储)的增删改查、连通性测试及租户默认存储设置。",
|
||||
capabilityManageWebSearch: "管理联网搜索",
|
||||
capabilityManageWebSearchHint: "允许管理联网搜索供应商配置、凭据和连接测试。",
|
||||
capabilityRunEvaluations: "运行评测",
|
||||
|
||||
@@ -645,6 +645,7 @@ const API_KEY_CAPABILITIES: TenantAPIKeyCapability[] = [
|
||||
'manage_datasources',
|
||||
'manage_models',
|
||||
'manage_vector_stores',
|
||||
'manage_storage_backends',
|
||||
'manage_web_search',
|
||||
'manage_channels',
|
||||
'run_evaluations',
|
||||
@@ -699,6 +700,7 @@ const apiKeyCapabilityGroups: ApiKeyCapabilityGroup[] = [
|
||||
capabilities: [
|
||||
{ value: 'manage_models', labelKey: 'integrations.api.capabilityManageModels', hintKey: 'integrations.api.capabilityManageModelsHint' },
|
||||
{ value: 'manage_vector_stores', labelKey: 'integrations.api.capabilityManageVectorStores', hintKey: 'integrations.api.capabilityManageVectorStoresHint' },
|
||||
{ value: 'manage_storage_backends', labelKey: 'integrations.api.capabilityManageStorageBackends', hintKey: 'integrations.api.capabilityManageStorageBackendsHint' },
|
||||
{ value: 'manage_web_search', labelKey: 'integrations.api.capabilityManageWebSearch', hintKey: 'integrations.api.capabilityManageWebSearchHint' },
|
||||
{ value: 'manage_channels', labelKey: 'integrations.api.capabilityManageChannels', hintKey: 'integrations.api.capabilityManageChannelsHint' },
|
||||
{ value: 'run_evaluations', labelKey: 'integrations.api.capabilityRunEvaluations', hintKey: 'integrations.api.capabilityRunEvaluationsHint' },
|
||||
@@ -786,6 +788,7 @@ function keyCapabilityLabels(key: TenantAPIKey): string[] {
|
||||
manage_datasources: t('integrations.api.capabilityManageDatasources'),
|
||||
manage_channels: t('integrations.api.capabilityManageChannels'),
|
||||
manage_vector_stores: t('integrations.api.capabilityManageVectorStores'),
|
||||
manage_storage_backends: t('integrations.api.capabilityManageStorageBackends'),
|
||||
manage_web_search: t('integrations.api.capabilityManageWebSearch'),
|
||||
run_evaluations: t('integrations.api.capabilityRunEvaluations'),
|
||||
manage_members: t('integrations.api.capabilityManageMembers'),
|
||||
|
||||
@@ -254,8 +254,10 @@
|
||||
<!-- 存储引擎 -->
|
||||
<div v-if="!isFAQ && formData && currentSection === 'storage'" class="section">
|
||||
<KBStorageSettings
|
||||
:storage-backend-id="formData.storageBackendId"
|
||||
:storage-provider="formData.storageProvider"
|
||||
:has-files="mode === 'edit' && hasFiles"
|
||||
@update:storage-backend-id="handleStorageBackendUpdate"
|
||||
@update:storage-provider="handleStorageProviderUpdate"
|
||||
/>
|
||||
</div>
|
||||
@@ -711,6 +713,7 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
|
||||
languages: [] as string[],
|
||||
tableMetadataInstructions: ''
|
||||
},
|
||||
storageBackendId: '' as string,
|
||||
storageProvider: '' as string,
|
||||
multimodalConfig: {
|
||||
enabled: false,
|
||||
@@ -832,6 +835,7 @@ const loadKBData = async () => {
|
||||
languages: kb.chunking_config?.languages || [],
|
||||
tableMetadataInstructions: kb.chunking_config?.table_metadata_instructions || ''
|
||||
},
|
||||
storageBackendId: (kb.storage_backend_id || '') as string,
|
||||
storageProvider: (kb.storage_provider_config?.provider || kb.storage_config?.provider || 'local') as string,
|
||||
multimodalConfig: {
|
||||
enabled: !!kb.vlm_config?.enabled,
|
||||
@@ -1025,6 +1029,12 @@ const handleStorageProviderUpdate = (value: string) => {
|
||||
}
|
||||
}
|
||||
|
||||
const handleStorageBackendUpdate = (value: string) => {
|
||||
if (formData.value) {
|
||||
formData.value.storageBackendId = value
|
||||
}
|
||||
}
|
||||
|
||||
async function loadTenantDefaultStorageProvider(force = false) {
|
||||
try {
|
||||
await editorResources.ensureStorageEngine(force)
|
||||
@@ -1177,8 +1187,11 @@ const buildSubmitData = () => {
|
||||
language: formData.value.asrConfig?.language || ''
|
||||
}
|
||||
|
||||
// 存储引擎:仅传 provider,参数从全局设置读取
|
||||
// Write to storage_provider_config (authoritative) + storage_config (legacy dual-write)
|
||||
// storage_backend_id is authoritative. Keep provider projection for old clients
|
||||
// and for rolling upgrades where a node has not picked up the new schema yet.
|
||||
if (formData.value.storageBackendId) {
|
||||
data.storage_backend_id = formData.value.storageBackendId
|
||||
}
|
||||
const storageProvider = resolvedStorageProvider()
|
||||
data.storage_provider_config = {
|
||||
provider: storageProvider
|
||||
@@ -1362,6 +1375,7 @@ const doSubmit = async () => {
|
||||
multimodal: {
|
||||
enabled: !!data.vlm_config?.enabled
|
||||
},
|
||||
storageBackendId: formData.value?.storageBackendId || '',
|
||||
storageProvider: data.storage_provider_config?.provider || data.storage_config?.provider || 'local',
|
||||
nodeExtract: {
|
||||
enabled: data.extract_config?.enabled || false,
|
||||
|
||||
@@ -2,48 +2,28 @@
|
||||
<div class="kb-storage-settings">
|
||||
<div class="section-header">
|
||||
<h2>{{ $t('kbSettings.storage.title') }}</h2>
|
||||
<p class="section-description">
|
||||
{{ $t('kbSettings.storage.description') }}
|
||||
</p>
|
||||
<p class="section-description">选择此知识库绑定的具体存储实例。</p>
|
||||
</div>
|
||||
|
||||
<div v-if="loading" class="loading-inline">
|
||||
<t-loading size="small" />
|
||||
<span>{{ $t('kbSettings.storage.loading') }}</span>
|
||||
</div>
|
||||
|
||||
<div v-if="loading" class="loading-inline"><t-loading size="small" /><span>加载中...</span></div>
|
||||
<div v-else class="settings-group">
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('kbSettings.storage.engineLabel') }}</label>
|
||||
<p class="desc">{{ $t('kbSettings.storage.engineDesc') }}</p>
|
||||
<label>存储实例</label>
|
||||
<p class="desc">同一种存储类型可以配置多个不同 Endpoint、Bucket 或凭据的实例。</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-select
|
||||
v-model="localProvider"
|
||||
size="medium"
|
||||
:placeholder="$t('kbSettings.storage.selectPlaceholder')"
|
||||
style="width: 100%; min-width: 220px;"
|
||||
@change="handleChange"
|
||||
>
|
||||
<t-option
|
||||
v-for="opt in engineOptions"
|
||||
:key="opt.value"
|
||||
:value="opt.value"
|
||||
:label="opt.label"
|
||||
:disabled="opt.disabled"
|
||||
>
|
||||
<t-select v-model="localID" style="width:100%;min-width:260px" :disabled="!!props.hasFiles" @change="handleChange">
|
||||
<t-option v-for="backend in backends" :key="backend.id" :value="backend.id" :label="backend.name">
|
||||
<span class="select-option">
|
||||
<span>{{ opt.label }}</span>
|
||||
<t-tag v-if="opt.disabled && opt.allowed === false" theme="danger" variant="light" size="small">{{ $t('kbSettings.storage.unavailable') }}</t-tag>
|
||||
<t-tag v-else-if="opt.disabled" theme="warning" variant="light" size="small">{{ $t('kbSettings.storage.notConfigured') }}</t-tag>
|
||||
<t-tag v-else-if="opt.available === false" theme="danger" variant="light" size="small">{{ $t('kbSettings.storage.unavailable') }}</t-tag>
|
||||
<span>{{ backend.name }}</span>
|
||||
<t-tag theme="primary" variant="light" size="small">{{ backend.provider.toUpperCase() }}</t-tag>
|
||||
<t-tag v-if="backend.id === defaultID" variant="light" size="small">默认</t-tag>
|
||||
</span>
|
||||
</t-option>
|
||||
</t-select>
|
||||
<p v-if="props.hasFiles" class="option-hint change-warning">{{ $t('kbSettings.storage.changeWarning') }}</p>
|
||||
<p v-else-if="selectedOption?.desc" class="option-hint">{{ selectedOption.desc }}</p>
|
||||
<a v-if="showGoSettings" href="javascript:void(0)" class="go-settings" @click.prevent="goToStorageSettings">{{ $t('kbSettings.storage.goGlobalSettings') }}</a>
|
||||
<p v-if="props.hasFiles" class="option-hint change-warning">知识库中已有文件,需通过存储迁移流程才能更换实例。</p>
|
||||
<p v-else-if="selected" class="option-hint">{{ selected.config.endpoint || selected.config.bucket_name || selected.config.path_prefix || '本地存储' }}</p>
|
||||
<a href="javascript:void(0)" class="go-settings" @click.prevent="goToSettings">管理存储实例</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -51,274 +31,39 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, watch, onMounted } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { type StorageEngineStatusItem } from '@/api/system'
|
||||
import { computed, onMounted, ref, watch } from 'vue'
|
||||
import { listStorageBackends, type StorageBackend } from '@/api/storage-backend'
|
||||
import { useUIStore } from '@/stores/ui'
|
||||
import { useEditorResourcesStore } from '@/stores/editorResources'
|
||||
|
||||
const { t } = useI18n()
|
||||
|
||||
const props = defineProps<{
|
||||
storageProvider: string
|
||||
hasFiles?: boolean
|
||||
}>()
|
||||
|
||||
const props = defineProps<{ storageBackendId?: string; storageProvider?: string; hasFiles?: boolean }>()
|
||||
const emit = defineEmits<{
|
||||
'update:storageBackendId': [value: string]
|
||||
'update:storageProvider': [value: string]
|
||||
}>()
|
||||
|
||||
const uiStore = useUIStore()
|
||||
const editorResources = useEditorResourcesStore()
|
||||
// Keep empty until tenant default_provider is loaded — do not pre-fill 'local'.
|
||||
const localProvider = ref(props.storageProvider)
|
||||
const loading = ref(true)
|
||||
const engineStatus = ref<StorageEngineStatusItem[]>([])
|
||||
const defaultProvider = ref('local')
|
||||
const allowedProviders = ref<string[]>([])
|
||||
const hasAnyConfig = ref(false)
|
||||
|
||||
const engineOptions = computed(() => {
|
||||
const statusByName = new Map(engineStatus.value.map((engine) => [engine.name, engine]))
|
||||
const allowedSet = new Set(allowedProviders.value)
|
||||
const hasStatus = engineStatus.value.length > 0
|
||||
const hasAllowedList = allowedProviders.value.length > 0
|
||||
const resolveState = (name: string, fallbackAvailable = false) => {
|
||||
const status = statusByName.get(name)
|
||||
const allowed = status?.allowed ?? (hasAllowedList ? allowedSet.has(name) : !hasStatus)
|
||||
const available = status?.available ?? fallbackAvailable
|
||||
return {
|
||||
allowed,
|
||||
available,
|
||||
disabled: !allowed || !available,
|
||||
}
|
||||
}
|
||||
|
||||
const local = resolveState('local', true)
|
||||
const minio = resolveState('minio')
|
||||
const cos = resolveState('cos')
|
||||
const tos = resolveState('tos')
|
||||
const s3 = resolveState('s3')
|
||||
const oss = resolveState('oss')
|
||||
const ks3 = resolveState('ks3')
|
||||
const obs = resolveState('obs')
|
||||
|
||||
return [
|
||||
{
|
||||
value: 'local',
|
||||
label: t('kbSettings.storage.engineLocal'),
|
||||
desc: t('kbSettings.storage.engineLocalDesc'),
|
||||
...local,
|
||||
},
|
||||
{
|
||||
value: 'minio',
|
||||
label: 'MinIO',
|
||||
desc: t('kbSettings.storage.engineMinioDesc'),
|
||||
...minio,
|
||||
},
|
||||
{
|
||||
value: 'cos',
|
||||
label: t('kbSettings.storage.engineCos'),
|
||||
desc: t('kbSettings.storage.engineCosDesc'),
|
||||
...cos,
|
||||
},
|
||||
{
|
||||
value: 'tos',
|
||||
label: t('kbSettings.storage.engineTos'),
|
||||
desc: t('kbSettings.storage.engineTosDesc'),
|
||||
...tos,
|
||||
},
|
||||
{
|
||||
value: 's3',
|
||||
label: t('kbSettings.storage.engineS3'),
|
||||
desc: t('kbSettings.storage.engineS3Desc'),
|
||||
...s3,
|
||||
},
|
||||
{
|
||||
value: 'oss',
|
||||
label: t('kbSettings.storage.engineOss'),
|
||||
desc: t('kbSettings.storage.engineOssDesc'),
|
||||
...oss,
|
||||
},
|
||||
{
|
||||
value: 'ks3',
|
||||
label: t('kbSettings.storage.engineKs3'),
|
||||
desc: t('kbSettings.storage.engineKs3Desc'),
|
||||
...ks3,
|
||||
},
|
||||
{
|
||||
value: 'obs',
|
||||
label: t('kbSettings.storage.engineObs'),
|
||||
desc: t('kbSettings.storage.engineObsDesc'),
|
||||
...obs,
|
||||
},
|
||||
]
|
||||
})
|
||||
|
||||
const showGoSettings = computed(() =>
|
||||
engineOptions.value.some(o => o.disabled)
|
||||
)
|
||||
|
||||
const selectedOption = computed(() =>
|
||||
engineOptions.value.find(o => o.value === localProvider.value)
|
||||
)
|
||||
const loading = ref(false), backends = ref<StorageBackend[]>([]), defaultID = ref(''), localID = ref(props.storageBackendId || '')
|
||||
const selected = computed(() => backends.value.find(item => item.id === localID.value))
|
||||
|
||||
function handleChange() {
|
||||
emit('update:storageProvider', localProvider.value)
|
||||
emit('update:storageBackendId', localID.value)
|
||||
emit('update:storageProvider', selected.value?.provider || props.storageProvider || '')
|
||||
}
|
||||
|
||||
function ensureAllowedProvider() {
|
||||
const resolved = editorResources.resolveUsableStorageProvider(localProvider.value)
|
||||
if (resolved === localProvider.value) return
|
||||
localProvider.value = resolved
|
||||
emit('update:storageProvider', localProvider.value)
|
||||
}
|
||||
|
||||
function goToStorageSettings() {
|
||||
uiStore.closeKBEditor?.()
|
||||
uiStore.openSettings?.('storage')
|
||||
}
|
||||
|
||||
async function load(force = false) {
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try {
|
||||
await editorResources.ensureStorageEngine(force)
|
||||
const engines = editorResources.storageStatus
|
||||
engineStatus.value = engines
|
||||
allowedProviders.value = editorResources.storageAllowedProviders
|
||||
defaultProvider.value = editorResources.resolveUsableStorageProvider(
|
||||
editorResources.storageConfig?.default_provider,
|
||||
)
|
||||
const d = editorResources.storageConfig
|
||||
hasAnyConfig.value = !!(d?.local?.path_prefix || d?.minio?.bucket_name || d?.cos?.bucket_name || d?.tos?.bucket_name || d?.s3?.bucket_name || d?.oss?.bucket_name || d?.ks3?.bucket_name || d?.obs?.bucket_name)
|
||||
const parentUnset = !props.storageProvider
|
||||
if (parentUnset) {
|
||||
localProvider.value = defaultProvider.value
|
||||
emit('update:storageProvider', localProvider.value)
|
||||
} else {
|
||||
localProvider.value = props.storageProvider
|
||||
}
|
||||
ensureAllowedProvider()
|
||||
} catch {
|
||||
engineStatus.value = []
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
const response = await listStorageBackends()
|
||||
backends.value = (response.data || []).filter(item => item.status === 'active')
|
||||
defaultID.value = response.default_storage_backend_id || ''
|
||||
if (!localID.value) localID.value = defaultID.value || backends.value[0]?.id || ''
|
||||
if (localID.value) handleChange()
|
||||
} finally { loading.value = false }
|
||||
}
|
||||
|
||||
// Sync only when parent sets an explicit provider (edit mode). Create mode leaves
|
||||
// storageProvider empty until load() applies tenant default_provider.
|
||||
watch(() => props.storageProvider, (v) => {
|
||||
if (v) {
|
||||
localProvider.value = v
|
||||
}
|
||||
})
|
||||
|
||||
function goToSettings() { uiStore.closeKBEditor?.(); uiStore.openSettings?.('storage') }
|
||||
watch(() => props.storageBackendId, value => { if (value) localID.value = value })
|
||||
onMounted(load)
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.kb-storage-settings {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.section-header {
|
||||
margin-bottom: 20px;
|
||||
|
||||
h2 {
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
color: var(--td-text-color-primary);
|
||||
margin: 0 0 6px 0;
|
||||
}
|
||||
|
||||
.section-description {
|
||||
font-size: 14px;
|
||||
color: var(--td-text-color-secondary);
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.loading-inline {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 16px 0;
|
||||
}
|
||||
|
||||
.settings-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.setting-row {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
padding: 16px 0;
|
||||
border-bottom: 1px solid var(--td-component-stroke);
|
||||
}
|
||||
|
||||
.setting-info {
|
||||
flex: 0 0 40%;
|
||||
max-width: 40%;
|
||||
padding-right: 24px;
|
||||
|
||||
label {
|
||||
font-size: 15px;
|
||||
font-weight: 500;
|
||||
color: var(--td-text-color-primary);
|
||||
display: block;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.desc {
|
||||
font-size: 13px;
|
||||
color: var(--td-text-color-secondary);
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.setting-control {
|
||||
flex: 0 0 55%;
|
||||
max-width: 55%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.select-option {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.option-hint {
|
||||
font-size: 12px;
|
||||
color: var(--td-text-color-placeholder);
|
||||
margin: 0;
|
||||
line-height: 1.4;
|
||||
|
||||
&.locked-hint {
|
||||
color: var(--td-warning-color);
|
||||
}
|
||||
|
||||
&.change-warning {
|
||||
color: var(--td-warning-color);
|
||||
}
|
||||
}
|
||||
|
||||
.go-settings {
|
||||
font-size: 13px;
|
||||
color: var(--td-brand-color, #0052d9);
|
||||
margin-top: 8px;
|
||||
text-decoration: none;
|
||||
|
||||
&:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
<style scoped lang="less">
|
||||
.section-header{margin-bottom:20px}.section-header h2{font-size:20px;margin:0 0 6px}.section-description,.desc,.option-hint{color:var(--td-text-color-secondary)}
|
||||
.setting-row{display:flex;justify-content:space-between;gap:28px}.setting-info{flex:1}.setting-control{width:45%;min-width:300px}.select-option{display:flex;align-items:center;gap:8px}.option-hint{font-size:12px;margin:8px 0}.change-warning{color:var(--td-warning-color)}.go-settings{font-size:13px;color:var(--td-brand-color)}.loading-inline{display:flex;gap:8px}
|
||||
</style>
|
||||
|
||||
@@ -196,7 +196,7 @@ import WebSearchSettings from './WebSearchSettings.vue'
|
||||
import ChatHistorySettings from './ChatHistorySettings.vue'
|
||||
import VectorStoreSettings from './VectorStoreSettings.vue'
|
||||
import ParserEngineSettings from './ParserEngineSettings.vue'
|
||||
import StorageEngineSettings from './StorageEngineSettings.vue'
|
||||
import StorageEngineSettings from './StorageBackendSettings.vue'
|
||||
import WeKnoraCloudSettings from './WeKnoraCloudSettings.vue'
|
||||
import TenantMembers from './TenantMembers.vue'
|
||||
import SystemSettings from '@/views/system/SystemSettings.vue'
|
||||
|
||||
@@ -0,0 +1,772 @@
|
||||
<template>
|
||||
<div class="storage-backend-settings">
|
||||
<div class="section-header">
|
||||
<div class="section-header__top">
|
||||
<div>
|
||||
<h2>{{ t('settings.storage.title') }}</h2>
|
||||
<p class="section-description">管理文件与图片使用的存储实例;同一种类型可以配置多个实例。</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<t-loading :loading="loading" size="small" class="backend-list-loading">
|
||||
<t-empty
|
||||
v-if="!loading && backends.length === 0 && !authStore.hasRole('admin')"
|
||||
description="尚未配置存储实例"
|
||||
/>
|
||||
<div v-else-if="!loading" class="backend-grid">
|
||||
<div
|
||||
v-for="backend in backends"
|
||||
:key="backend.id"
|
||||
class="backend-card"
|
||||
:class="[
|
||||
`backend-card--${backend.provider}`,
|
||||
{ 'backend-card--clickable': canEdit(backend) },
|
||||
]"
|
||||
:role="canEdit(backend) ? 'button' : undefined"
|
||||
:tabindex="canEdit(backend) ? 0 : undefined"
|
||||
@click="onCardClick($event, backend)"
|
||||
@keydown.enter="onCardClick($event, backend)"
|
||||
>
|
||||
<div
|
||||
class="backend-card__badge"
|
||||
:class="badgeClass(backend.provider)"
|
||||
:style="badgeStyle(backend.provider)"
|
||||
:aria-label="backend.provider"
|
||||
>
|
||||
<img
|
||||
v-if="resolveLogo(backend.provider)?.mode === 'color'"
|
||||
:src="resolveLogo(backend.provider)!.url"
|
||||
:alt="backend.provider"
|
||||
class="backend-card__badge-img"
|
||||
/>
|
||||
<template v-else-if="!resolveLogo(backend.provider)">{{ providerInitial(backend.provider) }}</template>
|
||||
</div>
|
||||
<div class="backend-card__body">
|
||||
<div class="backend-card__header">
|
||||
<h3 class="backend-card__title">{{ backend.name }}</h3>
|
||||
<t-tag v-if="backend.id === defaultID" theme="primary" variant="light" size="small">默认</t-tag>
|
||||
<div v-if="hasActions(backend)" class="backend-card__actions" @click.stop>
|
||||
<t-dropdown
|
||||
:options="getBackendOptions(backend)"
|
||||
placement="bottom-right"
|
||||
attach="body"
|
||||
trigger="click"
|
||||
@click="(data: any) => handleMenuAction(data.value, backend)"
|
||||
>
|
||||
<t-button variant="text" shape="square" size="small" class="backend-card__action-btn">
|
||||
<t-icon name="ellipsis" />
|
||||
</t-button>
|
||||
</t-dropdown>
|
||||
</div>
|
||||
</div>
|
||||
<p class="backend-card__subtitle">
|
||||
<span>{{ backend.provider.toUpperCase() }}</span>
|
||||
<template v-if="backendMeta(backend)">
|
||||
<span class="backend-card__sep">·</span>
|
||||
<span class="backend-card__meta">{{ backendMeta(backend) }}</span>
|
||||
</template>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button
|
||||
v-if="authStore.hasRole('admin')"
|
||||
type="button"
|
||||
class="backend-card backend-card--add"
|
||||
@click="openCreate"
|
||||
>
|
||||
<span class="backend-card--add__icon" aria-hidden="true">
|
||||
<add-icon />
|
||||
</span>
|
||||
<span class="backend-card--add__label">添加存储实例</span>
|
||||
</button>
|
||||
</div>
|
||||
</t-loading>
|
||||
|
||||
<SettingDrawer
|
||||
v-model:visible="visible"
|
||||
:title="editing ? '编辑存储实例' : '添加存储实例'"
|
||||
:class="`storage-backend-drawer storage-backend-drawer--${form.provider}`"
|
||||
:confirm-loading="saving"
|
||||
@confirm="save"
|
||||
@cancel="visible = false"
|
||||
>
|
||||
<template #headerIcon>
|
||||
<img
|
||||
v-if="currentLogo?.mode === 'color'"
|
||||
:src="currentLogo.url"
|
||||
:alt="form.provider"
|
||||
class="header-icon__img"
|
||||
/>
|
||||
<span
|
||||
v-else-if="currentLogo?.mode === 'mono'"
|
||||
class="header-icon__mono"
|
||||
:style="monoLogoStyle"
|
||||
/>
|
||||
<span v-else class="header-icon__text">{{ providerInitial(form.provider) }}</span>
|
||||
</template>
|
||||
<template #subtitle>
|
||||
<span>{{ editing ? '修改此存储实例的连接配置。' : '为文件与图片新增一个存储实例。' }}</span>
|
||||
</template>
|
||||
|
||||
<t-form :data="form" layout="vertical">
|
||||
<section class="setting-drawer__section">
|
||||
<h4 class="setting-drawer__section-title">基本信息</h4>
|
||||
<div class="form-item">
|
||||
<label class="form-label required">名称</label>
|
||||
<t-input v-model="form.name" placeholder="例如:生产 COS、归档 COS" clearable />
|
||||
</div>
|
||||
<div class="form-item">
|
||||
<label class="form-label required">存储类型</label>
|
||||
<t-select v-model="form.provider" :disabled="!!editing" @change="resetConfig">
|
||||
<t-option
|
||||
v-for="provider in providers"
|
||||
:key="provider"
|
||||
:value="provider"
|
||||
:label="provider.toUpperCase()"
|
||||
/>
|
||||
</t-select>
|
||||
</div>
|
||||
<div v-if="form.provider === 'minio'" class="form-item">
|
||||
<label class="form-label">部署模式</label>
|
||||
<div class="source-options" role="radiogroup">
|
||||
<button
|
||||
type="button"
|
||||
class="source-option"
|
||||
:class="{ 'is-active': form.config.mode !== 'docker' }"
|
||||
:disabled="!!editing"
|
||||
@click="form.config.mode = 'remote'"
|
||||
>
|
||||
<t-icon name="cloud" class="source-option__icon" />
|
||||
<span class="source-option__label">远程实例</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
class="source-option"
|
||||
:class="{ 'is-active': form.config.mode === 'docker' }"
|
||||
:disabled="!!editing"
|
||||
@click="form.config.mode = 'docker'"
|
||||
>
|
||||
<t-icon name="server" class="source-option__icon" />
|
||||
<span class="source-option__label">环境变量</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="setting-drawer__section">
|
||||
<h4 class="setting-drawer__section-title">连接配置</h4>
|
||||
<div v-if="needsEndpoint" class="form-item">
|
||||
<label class="form-label required">Endpoint</label>
|
||||
<t-input
|
||||
v-model="form.config.endpoint"
|
||||
:disabled="!!editing"
|
||||
:placeholder="form.provider === 'minio' ? 'storage.example.com:9000' : 'https://storage.example.com'"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div v-if="needsRegion" class="form-item">
|
||||
<label class="form-label required">Region</label>
|
||||
<t-input v-model="form.config.region" :disabled="!!editing" clearable />
|
||||
</div>
|
||||
<template v-if="needsCredentials">
|
||||
<div class="form-item">
|
||||
<label class="form-label required">Access Key / Secret ID</label>
|
||||
<t-input v-model="form.config.access_key_id" placeholder="***" clearable>
|
||||
<template #prefix-icon><t-icon name="lock-on" /></template>
|
||||
</t-input>
|
||||
</div>
|
||||
<div class="form-item">
|
||||
<label class="form-label required">Secret Key</label>
|
||||
<t-input v-model="form.config.secret_access_key" type="password" placeholder="***" clearable>
|
||||
<template #prefix-icon><t-icon name="lock-on" /></template>
|
||||
</t-input>
|
||||
</div>
|
||||
</template>
|
||||
<div v-if="form.provider !== 'local'" class="form-item">
|
||||
<label class="form-label required">Bucket</label>
|
||||
<t-input v-model="form.config.bucket_name" :disabled="!!editing" clearable />
|
||||
</div>
|
||||
<div v-if="form.provider === 'cos'" class="form-item">
|
||||
<label class="form-label">App ID</label>
|
||||
<t-input v-model="form.config.app_id" :disabled="!!editing" placeholder="可选" clearable />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="setting-drawer__section">
|
||||
<h4 class="setting-drawer__section-title">高级选项</h4>
|
||||
<div class="form-item">
|
||||
<label class="form-label">路径前缀</label>
|
||||
<t-input v-model="form.config.path_prefix" :disabled="!!editing" placeholder="weknora/" clearable />
|
||||
</div>
|
||||
<div v-if="form.provider === 'minio'" class="form-item">
|
||||
<div class="vision-toggle">
|
||||
<t-switch v-model="form.config.use_ssl" />
|
||||
<span class="form-desc form-desc--inline">使用 HTTPS 访问 MinIO</span>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="form.provider === 's3'" class="form-item">
|
||||
<div class="vision-toggle">
|
||||
<t-switch v-model="form.config.force_path_style" />
|
||||
<span class="form-desc form-desc--inline">使用 Path Style</span>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="form.provider === 'oss'" class="form-item">
|
||||
<div class="vision-toggle">
|
||||
<t-switch v-model="form.config.use_temp_bucket" />
|
||||
<span class="form-desc form-desc--inline">使用临时桶</span>
|
||||
</div>
|
||||
</div>
|
||||
<template v-if="['cos', 'tos'].includes(form.provider) || (form.provider === 'oss' && form.config.use_temp_bucket)">
|
||||
<div class="form-item">
|
||||
<label class="form-label">临时桶</label>
|
||||
<t-input v-model="form.config.temp_bucket_name" placeholder="可选,用于临时文件" clearable />
|
||||
</div>
|
||||
<div class="form-item">
|
||||
<label class="form-label">临时桶 Region</label>
|
||||
<t-input v-model="form.config.temp_region" placeholder="留空时使用主 Region" clearable />
|
||||
</div>
|
||||
</template>
|
||||
</section>
|
||||
</t-form>
|
||||
|
||||
<template #footer-left>
|
||||
<t-button variant="outline" :loading="testing" @click="testRaw">
|
||||
<template #icon>
|
||||
<t-icon v-if="!testing && rawTestResult === 'ok'" name="check-circle-filled" class="status-icon available" />
|
||||
<t-icon v-else-if="!testing && rawTestResult === 'error'" name="close-circle-filled" class="status-icon unavailable" />
|
||||
</template>
|
||||
测试连接
|
||||
</t-button>
|
||||
</template>
|
||||
</SettingDrawer>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { computed, onMounted, reactive, ref } from 'vue'
|
||||
import { DialogPlugin, MessagePlugin } from 'tdesign-vue-next'
|
||||
import { AddIcon } from 'tdesign-icons-vue-next'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { useAuthStore } from '@/stores/auth'
|
||||
import SettingDrawer from '@/components/settings/SettingDrawer.vue'
|
||||
import { providerLogo } from './providerLogos'
|
||||
import {
|
||||
createStorageBackend, deleteStorageBackend, listStorageBackends, listStorageBackendTypes,
|
||||
setDefaultStorageBackend, testStorageBackend, testStorageBackendByID, updateStorageBackend,
|
||||
type StorageBackend, type StorageBackendConfig,
|
||||
} from '@/api/storage-backend'
|
||||
|
||||
const { t } = useI18n()
|
||||
const authStore = useAuthStore()
|
||||
const loading = ref(false), saving = ref(false), testing = ref(false), visible = ref(false)
|
||||
const backends = ref<StorageBackend[]>([]), providers = ref<string[]>([]), defaultID = ref('')
|
||||
const editing = ref<StorageBackend | null>(null)
|
||||
const rawTestResult = ref<'ok' | 'error' | null>(null)
|
||||
const blankConfig = (): StorageBackendConfig => ({ mode: 'remote', endpoint: '', region: '', access_key_id: '', secret_access_key: '', bucket_name: '', path_prefix: '', use_ssl: true })
|
||||
const form = reactive<{ name: string; provider: string; config: StorageBackendConfig }>({ name: '', provider: 'local', config: blankConfig() })
|
||||
const needsEndpoint = computed(() => !['local', 'cos'].includes(form.provider) && !(form.provider === 'minio' && form.config.mode === 'docker'))
|
||||
const needsRegion = computed(() => !['local', 'minio'].includes(form.provider))
|
||||
const needsCredentials = computed(() => form.provider !== 'local' && !(form.provider === 'minio' && form.config.mode === 'docker'))
|
||||
|
||||
const resolveLogo = (provider: string) => providerLogo('storage', provider)
|
||||
const providerInitial = (provider: string) => (provider || '?').trim().charAt(0).toUpperCase() || '?'
|
||||
const badgeClass = (provider: string) => {
|
||||
const mode = resolveLogo(provider)?.mode
|
||||
return {
|
||||
'backend-card__badge--logo': !!mode,
|
||||
'backend-card__badge--color': mode === 'color',
|
||||
'backend-card__badge--mono': mode === 'mono',
|
||||
}
|
||||
}
|
||||
const badgeStyle = (provider: string): Record<string, string> => {
|
||||
const logo = resolveLogo(provider)
|
||||
return logo?.mode === 'mono' ? { '--logo-url': `url("${logo.url}")` } : {}
|
||||
}
|
||||
|
||||
const currentLogo = computed(() => providerLogo('storage', form.provider))
|
||||
const monoLogoStyle = computed((): Record<string, string> => {
|
||||
const logo = currentLogo.value
|
||||
if (!logo || logo.mode !== 'mono') return {}
|
||||
return { '--logo-url': `url("${logo.url}")` }
|
||||
})
|
||||
|
||||
function backendMeta(backend: StorageBackend): string {
|
||||
return backend.config.endpoint || backend.config.bucket_name || backend.config.path_prefix || '本地存储'
|
||||
}
|
||||
|
||||
const canEdit = (backend: StorageBackend) => authStore.hasRole('admin') && backend.source !== 'env'
|
||||
const canDelete = (backend: StorageBackend) => authStore.hasRole('admin') && backend.source !== 'env' && !backend.legacy_alias
|
||||
const canSetDefault = (backend: StorageBackend) => backend.id !== defaultID.value && authStore.hasRole('admin')
|
||||
// 测试连接对所有可见用户开放,因此每张卡至少有一个动作。
|
||||
const hasActions = (_backend: StorageBackend) => true
|
||||
|
||||
function getBackendOptions(backend: StorageBackend) {
|
||||
const options: { content: string; value: string; theme?: string }[] = []
|
||||
options.push({ content: '测试连接', value: 'test' })
|
||||
if (canSetDefault(backend)) options.push({ content: '设为默认', value: 'default' })
|
||||
if (canEdit(backend)) options.push({ content: '编辑', value: 'edit' })
|
||||
if (canDelete(backend)) options.push({ content: '删除', value: 'delete', theme: 'error' })
|
||||
return options
|
||||
}
|
||||
|
||||
function handleMenuAction(value: string, backend: StorageBackend) {
|
||||
if (value === 'test') testSaved(backend)
|
||||
else if (value === 'default') makeDefault(backend)
|
||||
else if (value === 'edit') openEdit(backend)
|
||||
else if (value === 'delete') remove(backend)
|
||||
}
|
||||
|
||||
function onCardClick(event: Event, backend: StorageBackend) {
|
||||
if (!canEdit(backend)) return
|
||||
const target = event.target as HTMLElement | null
|
||||
if (target?.closest('.backend-card__actions')) return
|
||||
openEdit(backend)
|
||||
}
|
||||
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try {
|
||||
const [list, types] = await Promise.all([listStorageBackends(), listStorageBackendTypes()])
|
||||
backends.value = list.data || []; defaultID.value = list.default_storage_backend_id || ''; providers.value = types.data || []
|
||||
} finally { loading.value = false }
|
||||
}
|
||||
function resetConfig() { form.config = blankConfig(); rawTestResult.value = null }
|
||||
function openCreate() { editing.value = null; form.name = ''; form.provider = providers.value[0] || 'local'; form.config = blankConfig(); rawTestResult.value = null; visible.value = true }
|
||||
function openEdit(backend: StorageBackend) { editing.value = backend; form.name = backend.name; form.provider = backend.provider; form.config = { ...blankConfig(), ...backend.config }; rawTestResult.value = null; visible.value = true }
|
||||
async function testRaw() {
|
||||
testing.value = true
|
||||
rawTestResult.value = null
|
||||
try {
|
||||
const r: any = editing.value ? await testStorageBackendByID(editing.value.id) : await testStorageBackend(form)
|
||||
if (r.success) { rawTestResult.value = 'ok'; MessagePlugin.success('连接成功') }
|
||||
else { rawTestResult.value = 'error'; MessagePlugin.error(r.error || '连接失败') }
|
||||
} finally { testing.value = false }
|
||||
}
|
||||
async function testSaved(backend: StorageBackend) { const r: any = await testStorageBackendByID(backend.id); r.success ? MessagePlugin.success('连接成功') : MessagePlugin.error(r.error || '连接失败') }
|
||||
async function save() {
|
||||
if (!form.name.trim()) { MessagePlugin.warning('请输入名称'); return }
|
||||
saving.value = true
|
||||
try {
|
||||
const payload = { name: form.name.trim(), provider: form.provider, config: { ...form.config } }
|
||||
if (editing.value) await updateStorageBackend(editing.value.id, payload); else await createStorageBackend(payload)
|
||||
MessagePlugin.success('保存成功'); visible.value = false; await load()
|
||||
} catch (e: any) { MessagePlugin.error(e?.message || '保存失败') } finally { saving.value = false }
|
||||
}
|
||||
async function makeDefault(backend: StorageBackend) { await setDefaultStorageBackend(backend.id); defaultID.value = backend.id; MessagePlugin.success('默认存储已更新') }
|
||||
function remove(backend: StorageBackend) {
|
||||
const dialog = DialogPlugin.confirm({ header: '删除存储实例', body: `确定删除“${backend.name}”吗?`, onConfirm: async () => { dialog.destroy(); try { await deleteStorageBackend(backend.id); await load(); MessagePlugin.success('已删除') } catch (e: any) { MessagePlugin.error(e?.message || '删除失败') } }, onCancel: () => dialog.destroy() })
|
||||
}
|
||||
onMounted(load)
|
||||
</script>
|
||||
|
||||
<style scoped lang="less">
|
||||
.storage-backend-settings {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.section-header {
|
||||
margin-bottom: 28px;
|
||||
|
||||
h2 {
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
color: var(--td-text-color-primary);
|
||||
margin: 0 0 8px 0;
|
||||
}
|
||||
|
||||
.section-description {
|
||||
font-size: 14px;
|
||||
color: var(--td-text-color-secondary);
|
||||
margin: 0;
|
||||
line-height: 1.6;
|
||||
}
|
||||
}
|
||||
|
||||
.section-header__top {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 20px;
|
||||
}
|
||||
|
||||
.backend-list-loading {
|
||||
min-height: 120px;
|
||||
}
|
||||
|
||||
.backend-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(320px, 1fr));
|
||||
gap: 12px;
|
||||
|
||||
.backend-card--add {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.backend-card {
|
||||
position: relative;
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 12px;
|
||||
padding: 14px 16px;
|
||||
border: 1px solid var(--td-component-stroke);
|
||||
border-radius: 10px;
|
||||
background: var(--td-bg-color-container);
|
||||
transition: border-color 0.18s ease, box-shadow 0.18s ease;
|
||||
min-width: 0;
|
||||
|
||||
&:hover {
|
||||
border-color: var(--td-brand-color-3, var(--td-brand-color));
|
||||
box-shadow: 0 4px 14px rgba(15, 23, 42, 0.06);
|
||||
}
|
||||
|
||||
&--clickable {
|
||||
cursor: pointer;
|
||||
|
||||
&:focus-visible {
|
||||
outline: 2px solid var(--td-brand-color);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
}
|
||||
|
||||
&--add {
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
min-height: 68px;
|
||||
border-style: dashed;
|
||||
background: transparent;
|
||||
color: var(--td-text-color-placeholder);
|
||||
cursor: pointer;
|
||||
font: inherit;
|
||||
text-align: center;
|
||||
|
||||
&:hover,
|
||||
&:focus-visible {
|
||||
color: var(--td-brand-color);
|
||||
border-color: var(--td-brand-color);
|
||||
background: color-mix(in srgb, var(--td-brand-color) 6%, transparent);
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
&:focus-visible {
|
||||
outline: 2px solid var(--td-brand-color);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
&__icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 8px;
|
||||
background: color-mix(in srgb, var(--td-brand-color) 10%, transparent);
|
||||
color: var(--td-brand-color);
|
||||
font-size: 18px;
|
||||
}
|
||||
|
||||
&__label {
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
line-height: 1.4;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.backend-card__badge {
|
||||
flex-shrink: 0;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
border-radius: 9px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
margin-top: 1px;
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.02em;
|
||||
background: rgba(0, 82, 217, 0.1);
|
||||
color: #0052D9;
|
||||
}
|
||||
|
||||
.backend-card .backend-card__badge--logo {
|
||||
background: var(--td-bg-color-container, #fff);
|
||||
box-shadow: inset 0 0 0 1px var(--td-component-stroke);
|
||||
}
|
||||
|
||||
.backend-card .backend-card__badge--mono::before {
|
||||
content: '';
|
||||
width: 22px;
|
||||
height: 22px;
|
||||
background-color: currentColor;
|
||||
-webkit-mask-image: var(--logo-url);
|
||||
-webkit-mask-position: center;
|
||||
-webkit-mask-repeat: no-repeat;
|
||||
-webkit-mask-size: contain;
|
||||
mask-image: var(--logo-url);
|
||||
mask-position: center;
|
||||
mask-repeat: no-repeat;
|
||||
mask-size: contain;
|
||||
}
|
||||
|
||||
.backend-card__badge-img {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
object-fit: contain;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.backend-card--local .backend-card__badge { background: rgba(70, 70, 70, 0.1); color: #464646; }
|
||||
.backend-card--minio .backend-card__badge { background: rgba(225, 38, 38, 0.12); color: #C0382B; }
|
||||
.backend-card--cos .backend-card__badge { background: rgba(0, 82, 217, 0.1); color: #0052D9; }
|
||||
.backend-card--tos .backend-card__badge { background: rgba(0, 137, 255, 0.12); color: #0089FF; }
|
||||
.backend-card--s3 .backend-card__badge { background: rgba(255, 153, 0, 0.12); color: #D97706; }
|
||||
.backend-card--oss .backend-card__badge { background: rgba(255, 90, 0, 0.12); color: #E55A00; }
|
||||
.backend-card--ks3 .backend-card__badge { background: rgba(7, 192, 95, 0.12); color: #07A050; }
|
||||
.backend-card--obs .backend-card__badge { background: rgba(206, 17, 38, 0.1); color: #CE1126; }
|
||||
|
||||
.backend-card__body {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: center;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.backend-card__header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.backend-card__title {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
margin: 0;
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
line-height: 1.4;
|
||||
color: var(--td-text-color-primary);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.backend-card__subtitle {
|
||||
margin: 2px 0 0;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
color: var(--td-text-color-secondary);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.backend-card__sep {
|
||||
margin: 0 6px;
|
||||
color: var(--td-text-color-placeholder);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.backend-card__meta {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.backend-card__actions {
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.backend-card__action-btn {
|
||||
flex-shrink: 0;
|
||||
padding: 2px;
|
||||
color: var(--td-text-color-placeholder);
|
||||
opacity: 0;
|
||||
transition: opacity 0.15s ease;
|
||||
|
||||
&:hover,
|
||||
&:focus-visible {
|
||||
background: var(--td-bg-color-secondarycontainer);
|
||||
color: var(--td-text-color-primary);
|
||||
}
|
||||
}
|
||||
|
||||
.backend-card:hover .backend-card__action-btn,
|
||||
.backend-card:focus-within .backend-card__action-btn {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
// ---- 抽屉头部图标 ----
|
||||
.header-icon__img {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
object-fit: contain;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.header-icon__mono {
|
||||
display: inline-block;
|
||||
width: 22px;
|
||||
height: 22px;
|
||||
background-color: currentColor;
|
||||
-webkit-mask-image: var(--logo-url);
|
||||
-webkit-mask-position: center;
|
||||
-webkit-mask-repeat: no-repeat;
|
||||
-webkit-mask-size: contain;
|
||||
mask-image: var(--logo-url);
|
||||
mask-position: center;
|
||||
mask-repeat: no-repeat;
|
||||
mask-size: contain;
|
||||
}
|
||||
|
||||
.header-icon__text {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.02em;
|
||||
}
|
||||
|
||||
// ---- 抽屉表单 ----
|
||||
.form-item {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.form-label {
|
||||
display: block;
|
||||
margin-bottom: 6px;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: var(--td-text-color-primary);
|
||||
line-height: 1.4;
|
||||
|
||||
&.required::before {
|
||||
content: '*';
|
||||
color: var(--td-error-color);
|
||||
margin-right: 4px;
|
||||
font-weight: 500;
|
||||
line-height: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.form-desc {
|
||||
margin: 4px 0 0 0;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
color: var(--td-text-color-placeholder);
|
||||
|
||||
&--inline {
|
||||
margin: 0;
|
||||
}
|
||||
}
|
||||
|
||||
:deep(.t-input),
|
||||
:deep(.t-select),
|
||||
:deep(.t-textarea) {
|
||||
width: 100%;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.vision-toggle {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
// ---- MinIO 部署模式 pill segmented ----
|
||||
.source-options {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
padding: 3px;
|
||||
background: var(--td-bg-color-component);
|
||||
border: 1px solid var(--td-component-stroke);
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.source-option {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 5px 12px;
|
||||
height: 28px;
|
||||
background: transparent;
|
||||
border: 1px solid transparent;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
font-family: inherit;
|
||||
font-size: 13px;
|
||||
color: var(--td-text-color-secondary);
|
||||
line-height: 1;
|
||||
transition: all 0.15s ease;
|
||||
|
||||
&:disabled {
|
||||
cursor: not-allowed;
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
&:hover:not(.is-active):not(:disabled) {
|
||||
color: var(--td-text-color-primary);
|
||||
background: var(--td-bg-color-container-hover);
|
||||
}
|
||||
|
||||
&.is-active {
|
||||
background: var(--td-bg-color-container);
|
||||
border-color: var(--td-brand-color);
|
||||
color: var(--td-brand-color);
|
||||
font-weight: 500;
|
||||
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.04);
|
||||
}
|
||||
}
|
||||
|
||||
.source-option__icon {
|
||||
font-size: 14px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.source-option__label {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.status-icon {
|
||||
font-size: 16px;
|
||||
flex-shrink: 0;
|
||||
|
||||
&.available {
|
||||
color: var(--td-brand-color);
|
||||
}
|
||||
|
||||
&.unavailable {
|
||||
color: var(--td-error-color);
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
<!--
|
||||
Non-scoped: drawer header icon coloring per provider, mirroring the list
|
||||
card badge colors. Namespaced under .storage-backend-drawer--{id}.
|
||||
-->
|
||||
<style lang="less">
|
||||
.storage-backend-drawer .setting-drawer__header-icon:has(.header-icon__img) {
|
||||
background: var(--td-bg-color-container, #fff);
|
||||
box-shadow: inset 0 0 0 1px var(--td-component-stroke);
|
||||
}
|
||||
|
||||
.storage-backend-drawer--local .setting-drawer__header-icon { background: rgba(70, 70, 70, 0.1); color: #464646; }
|
||||
.storage-backend-drawer--minio .setting-drawer__header-icon { background: rgba(225, 38, 38, 0.12); color: #C0382B; }
|
||||
.storage-backend-drawer--cos .setting-drawer__header-icon { background: rgba(0, 82, 217, 0.1); color: #0052D9; }
|
||||
.storage-backend-drawer--tos .setting-drawer__header-icon { background: rgba(0, 137, 255, 0.12); color: #0089FF; }
|
||||
.storage-backend-drawer--s3 .setting-drawer__header-icon { background: rgba(255, 153, 0, 0.12); color: #D97706; }
|
||||
.storage-backend-drawer--oss .setting-drawer__header-icon { background: rgba(255, 90, 0, 0.12); color: #E55A00; }
|
||||
.storage-backend-drawer--ks3 .setting-drawer__header-icon { background: rgba(7, 192, 95, 0.12); color: #07A050; }
|
||||
.storage-backend-drawer--obs .setting-drawer__header-icon { background: rgba(206, 17, 38, 0.1); color: #CE1126; }
|
||||
</style>
|
||||
@@ -113,7 +113,7 @@
|
||||
- color logo(如 MinIO/AWS): 直接 <img>,保留品牌彩色
|
||||
- mono logo(mask-image): 通过 ::before + currentColor 染色,颜色由
|
||||
.storage-engine-drawer--{id} :deep(.setting-drawer__header-icon) 决定
|
||||
- 无 logo(local): 渲染首字母作为 monogram
|
||||
- 无 logo: 渲染首字母作为 monogram
|
||||
-->
|
||||
<template v-if="currentEngine" #headerIcon>
|
||||
<img
|
||||
|
||||
@@ -122,7 +122,8 @@ type DataAnalysisTool struct {
|
||||
// env var at request time can produce a different (or empty) value if the
|
||||
// variable was not exported to the sub-process or was set programmatically
|
||||
// after startup, causing GetFile to look in the wrong directory (#1040).
|
||||
localBaseDir string
|
||||
localBaseDir string
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
}
|
||||
|
||||
func NewDataAnalysisTool(
|
||||
@@ -132,8 +133,9 @@ func NewDataAnalysisTool(
|
||||
fileService interfaces.FileService,
|
||||
db *sql.DB,
|
||||
sessionID string,
|
||||
storageResolvers ...interfaces.StorageBackendResolver,
|
||||
) *DataAnalysisTool {
|
||||
return &DataAnalysisTool{
|
||||
tool := &DataAnalysisTool{
|
||||
BaseTool: dataAnalysisTool,
|
||||
knowledgeBaseService: knowledgeBaseService,
|
||||
knowledgeService: knowledgeService,
|
||||
@@ -145,8 +147,12 @@ func NewDataAnalysisTool(
|
||||
// call to resolveFileServiceForKnowledge uses the same base path. The
|
||||
// env var is guaranteed to be set (or empty == "/data/files" fallback)
|
||||
// when the application starts and the DI container is assembled.
|
||||
localBaseDir: strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")),
|
||||
localBaseDir: strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")),
|
||||
}
|
||||
if len(storageResolvers) > 0 {
|
||||
tool.storageResolver = storageResolvers[0]
|
||||
}
|
||||
return tool
|
||||
}
|
||||
|
||||
// recordCreatedTable records a table name for cleanup, ensuring uniqueness
|
||||
@@ -782,8 +788,12 @@ func (t *DataAnalysisTool) resolveFileServiceForKnowledge(ctx context.Context, k
|
||||
}
|
||||
|
||||
provider := ""
|
||||
backendID, _, _ := types.ParseStorageBackendPath(knowledge.FilePath)
|
||||
if kb != nil {
|
||||
provider = kb.GetStorageProvider()
|
||||
if backendID == "" && kb.StorageBackendID != nil {
|
||||
backendID = strings.TrimSpace(*kb.StorageBackendID)
|
||||
}
|
||||
}
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant == nil {
|
||||
@@ -809,6 +819,18 @@ func (t *DataAnalysisTool) resolveFileServiceForKnowledge(ctx context.Context, k
|
||||
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
|
||||
}
|
||||
if t.storageResolver != nil && tenant != nil && (backendID != "" || provider != "") {
|
||||
resolvedSvc, resolvedProvider, err := t.storageResolver.ResolveFileService(
|
||||
ctx, tenant, backendID, provider, t.localBaseDir,
|
||||
)
|
||||
if err == nil {
|
||||
logger.Infof(ctx, "[Tool][DataAnalysis][storage] resolved storage backend: session_id=%s knowledge_id=%s kb_id=%s backend_id=%s provider=%s",
|
||||
t.sessionID, knowledge.ID, kbID, backendID, resolvedProvider)
|
||||
return resolvedSvc
|
||||
}
|
||||
logger.Warnf(ctx, "[Tool][DataAnalysis][storage] resolve storage backend failed, trying legacy config: session_id=%s knowledge_id=%s kb_id=%s backend_id=%s provider=%s err=%v",
|
||||
t.sessionID, knowledge.ID, kbID, backendID, provider, err)
|
||||
}
|
||||
|
||||
if provider == "" || tenant == nil || tenant.StorageEngineConfig == nil {
|
||||
hasTenantStorageConfig := tenant != nil && tenant.StorageEngineConfig != nil
|
||||
|
||||
@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
|
||||
pinned_at DATETIME NULL,
|
||||
asr_config TEXT,
|
||||
vector_store_id VARCHAR(36),
|
||||
storage_backend_id VARCHAR(36),
|
||||
wiki_config TEXT,
|
||||
indexing_strategy TEXT,
|
||||
creator_id VARCHAR(36),
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type storageBackendRepository struct{ db *gorm.DB }
|
||||
|
||||
func NewStorageBackendRepository(db *gorm.DB) interfaces.StorageBackendRepository {
|
||||
return &storageBackendRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *storageBackendRepository) Create(ctx context.Context, backend *types.StorageBackend) error {
|
||||
return r.db.WithContext(ctx).Create(backend).Error
|
||||
}
|
||||
|
||||
func (r *storageBackendRepository) GetByID(ctx context.Context, tenantID uint64, id string) (*types.StorageBackend, error) {
|
||||
var backend types.StorageBackend
|
||||
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).First(&backend).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &backend, nil
|
||||
}
|
||||
|
||||
func (r *storageBackendRepository) List(ctx context.Context, tenantID uint64) ([]*types.StorageBackend, error) {
|
||||
var backends []*types.StorageBackend
|
||||
err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).Order("created_at DESC").Find(&backends).Error
|
||||
return backends, err
|
||||
}
|
||||
|
||||
func (r *storageBackendRepository) Update(ctx context.Context, backend *types.StorageBackend) error {
|
||||
return r.db.WithContext(ctx).Model(&types.StorageBackend{}).
|
||||
Where("tenant_id = ? AND id = ?", backend.TenantID, backend.ID).
|
||||
Select("name", "config", "status", "updated_at").Updates(backend).Error
|
||||
}
|
||||
|
||||
func (r *storageBackendRepository) Delete(ctx context.Context, tenantID uint64, id string) error {
|
||||
return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.StorageBackend{}).Error
|
||||
}
|
||||
|
||||
func (r *storageBackendRepository) FindLegacyAlias(ctx context.Context, tenantID uint64, provider string) (*types.StorageBackend, error) {
|
||||
var backend types.StorageBackend
|
||||
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND provider = ? AND legacy_alias = ?", tenantID, provider, true).First(&backend).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &backend, nil
|
||||
}
|
||||
@@ -100,6 +100,7 @@ type agentService struct {
|
||||
webSearchStateService interfaces.WebSearchStateService
|
||||
wikiPageService interfaces.WikiPageService
|
||||
tenantService interfaces.TenantService
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
toolApprovalGate approval.MCPApproval
|
||||
}
|
||||
|
||||
@@ -120,6 +121,7 @@ func NewAgentService(
|
||||
webSearchStateService interfaces.WebSearchStateService,
|
||||
wikiPageService interfaces.WikiPageService,
|
||||
tenantService interfaces.TenantService,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
toolApprovalGate approval.MCPApproval,
|
||||
) interfaces.AgentService {
|
||||
return &agentService{
|
||||
@@ -138,6 +140,7 @@ func NewAgentService(
|
||||
webSearchStateService: webSearchStateService,
|
||||
wikiPageService: wikiPageService,
|
||||
tenantService: tenantService,
|
||||
storageResolver: storageResolver,
|
||||
toolApprovalGate: toolApprovalGate,
|
||||
}
|
||||
}
|
||||
@@ -622,7 +625,7 @@ func (s *agentService) registerTools(
|
||||
logger.Infof(ctx, "Registered web_fetch tool for session: %s", sessionID)
|
||||
|
||||
case tools.ToolDataAnalysis:
|
||||
toolToRegister = tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, s.fileService, s.duckdb, sessionID)
|
||||
toolToRegister = tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, s.fileService, s.duckdb, sessionID, s.storageResolver)
|
||||
logger.Infof(ctx, "Registered data_analysis tool for session: %s", sessionID)
|
||||
|
||||
case tools.ToolDataSchema:
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/agent/tools"
|
||||
chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline"
|
||||
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
@@ -406,6 +405,7 @@ type DataTableSummaryService struct {
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ownership retriever.TenantStoreOwnership
|
||||
sqlDB *sql.DB
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
}
|
||||
|
||||
// NewDataTableSummaryService creates a new DataTableSummaryService
|
||||
@@ -419,6 +419,7 @@ func NewDataTableSummaryService(
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ownership retriever.TenantStoreOwnership,
|
||||
sqlDB *sql.DB,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
) interfaces.TaskHandler {
|
||||
return &DataTableSummaryService{
|
||||
modelService: modelService,
|
||||
@@ -430,6 +431,7 @@ func NewDataTableSummaryService(
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
sqlDB: sqlDB,
|
||||
storageResolver: storageResolver,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -559,24 +561,29 @@ func (s *DataTableSummaryService) resolveFileServiceForKnowledge(ctx context.Con
|
||||
if resources == nil || resources.knowledge == nil {
|
||||
return s.fileService
|
||||
}
|
||||
if resources.tenant == nil || resources.tenant.StorageEngineConfig == nil {
|
||||
if resources.tenant == nil {
|
||||
return s.fileService
|
||||
}
|
||||
|
||||
provider := types.InferStorageFromFilePath(resources.knowledge.FilePath)
|
||||
if provider == "" {
|
||||
if provider == "" && resources.tenant.StorageEngineConfig != nil {
|
||||
provider = strings.ToLower(strings.TrimSpace(resources.tenant.StorageEngineConfig.DefaultProvider))
|
||||
}
|
||||
if provider == "" {
|
||||
return s.fileService
|
||||
}
|
||||
|
||||
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
||||
resolvedSvc, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig(
|
||||
provider,
|
||||
resources.tenant.StorageEngineConfig,
|
||||
baseDir,
|
||||
)
|
||||
backendID, _, _ := types.ParseStorageBackendPath(resources.knowledge.FilePath)
|
||||
if backendID == "" && resources.knowledgeBase != nil && resources.knowledgeBase.StorageBackendID != nil {
|
||||
backendID = strings.TrimSpace(*resources.knowledgeBase.StorageBackendID)
|
||||
}
|
||||
|
||||
// New-model workspaces resolve via DefaultStorageBackendID even when no
|
||||
// legacy StorageEngineConfig / provider is present, so gate on the resolver
|
||||
// and a usable backendID/provider rather than requiring a non-empty provider.
|
||||
if s.storageResolver == nil || (backendID == "" && provider == "") {
|
||||
return s.fileService
|
||||
}
|
||||
|
||||
resolvedSvc, resolvedProvider, err := s.storageResolver.ResolveFileService(ctx, resources.tenant, backendID, provider, baseDir)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[TableSummary] Failed to resolve file service for provider=%s, fallback to default: %v", provider, err)
|
||||
return s.fileService
|
||||
@@ -591,7 +598,7 @@ func (s *DataTableSummaryService) processTableData(ctx context.Context, resource
|
||||
// 创建DuckDB会话并加载数据
|
||||
sessionID := fmt.Sprintf("table_summary_%s", resources.knowledge.ID)
|
||||
fileSvc := s.resolveFileServiceForKnowledge(ctx, resources)
|
||||
duckdbTool := tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, fileSvc, s.sqlDB, sessionID)
|
||||
duckdbTool := tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, fileSvc, s.sqlDB, sessionID, s.storageResolver)
|
||||
defer duckdbTool.Cleanup(ctx)
|
||||
|
||||
// 使用knowledge.ID作为表名,根据文件类型自动加载数据
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// backendScopedFileService makes the storage instance part of every newly
|
||||
// persisted path while delegating actual I/O to the existing provider driver.
|
||||
type backendScopedFileService struct {
|
||||
backendID string
|
||||
inner interfaces.FileService
|
||||
}
|
||||
|
||||
func NewBackendScopedFileService(backendID string, inner interfaces.FileService) interfaces.FileService {
|
||||
return &backendScopedFileService{backendID: backendID, inner: inner}
|
||||
}
|
||||
|
||||
func (s *backendScopedFileService) unwrap(path string) (string, error) {
|
||||
id, inner, ok := types.ParseStorageBackendPath(path)
|
||||
if !ok {
|
||||
return path, nil
|
||||
}
|
||||
if id != s.backendID {
|
||||
return "", fmt.Errorf("storage backend mismatch")
|
||||
}
|
||||
return inner, nil
|
||||
}
|
||||
|
||||
func (s *backendScopedFileService) wrap(path string) string {
|
||||
return types.BuildStorageBackendPath(s.backendID, path)
|
||||
}
|
||||
func (s *backendScopedFileService) CheckConnectivity(ctx context.Context) error {
|
||||
return s.inner.CheckConnectivity(ctx)
|
||||
}
|
||||
func (s *backendScopedFileService) SaveFile(ctx context.Context, f *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
|
||||
p, err := s.inner.SaveFile(ctx, f, tenantID, knowledgeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.wrap(p), nil
|
||||
}
|
||||
func (s *backendScopedFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, name string, temp bool) (string, error) {
|
||||
p, err := s.inner.SaveBytes(ctx, data, tenantID, name, temp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.wrap(p), nil
|
||||
}
|
||||
func (s *backendScopedFileService) GetFile(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||
p, err := s.unwrap(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.inner.GetFile(ctx, p)
|
||||
}
|
||||
func (s *backendScopedFileService) GetFileURL(ctx context.Context, path string) (string, error) {
|
||||
p, err := s.unwrap(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
result, err := s.inner.GetFileURL(ctx, p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
scoped := s.wrap(p)
|
||||
if result == p {
|
||||
return scoped, nil
|
||||
}
|
||||
// Local storage may return an app-level presigned URL. Re-sign it with
|
||||
// the scoped path so the proxy resolves the exact local instance instead
|
||||
// of falling back to another backend of the same provider.
|
||||
if u, parseErr := url.Parse(result); parseErr == nil && strings.HasSuffix(u.Path, "/api/v1/files/presigned") && u.Query().Get("file_path") == p {
|
||||
basePath := strings.TrimSuffix(u.Path, "/api/v1/files/presigned")
|
||||
baseURL := u.Scheme + "://" + u.Host + basePath
|
||||
ttl := time.Duration(0)
|
||||
if expires, convErr := strconv.ParseInt(u.Query().Get("expires"), 10, 64); convErr == nil {
|
||||
ttl = time.Until(time.Unix(expires, 0))
|
||||
}
|
||||
if signed, signErr := secutils.SignFileURL(baseURL, scoped, secutils.ParseTenantIDFromStoragePath(scoped), ttl); signErr == nil {
|
||||
return signed, nil
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
func (s *backendScopedFileService) DeleteFile(ctx context.Context, path string) error {
|
||||
p, err := s.unwrap(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.inner.DeleteFile(ctx, p)
|
||||
}
|
||||
func (s *backendScopedFileService) CopyFile(ctx context.Context, path string, tenantID uint64, knowledgeID string) (string, error) {
|
||||
p, err := s.unwrap(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
result, err := s.inner.CopyFile(ctx, p, tenantID, knowledgeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.wrap(result), nil
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBackendScopedLocalURLRetainsBackendID(t *testing.T) {
|
||||
t.Setenv("SYSTEM_AES_KEY", "0123456789abcdef0123456789abcdef")
|
||||
inner := NewLocalFileService(t.TempDir(), "https://weknora.example.com/base")
|
||||
svc := NewBackendScopedFileService("backend-local-a", inner)
|
||||
|
||||
path, err := svc.SaveBytes(context.Background(), []byte("hello"), 7, "exports/image.txt", false)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, path, "storage://backend-local-a/local://")
|
||||
|
||||
signed, err := svc.GetFileURL(context.Background(), path)
|
||||
require.NoError(t, err)
|
||||
u, err := url.Parse(signed)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, path, u.Query().Get("file_path"))
|
||||
|
||||
reader, err := svc.GetFile(context.Background(), path)
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
data, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello", string(data))
|
||||
}
|
||||
@@ -82,14 +82,14 @@ func NewFileServiceFromStorageConfig(
|
||||
if pathPrefix == "" {
|
||||
pathPrefix = "weknora"
|
||||
}
|
||||
svc, err := NewCosFileService(sec.COS.BucketName, sec.COS.Region, sec.COS.SecretID, sec.COS.SecretKey, pathPrefix)
|
||||
svc, err := NewCosFileServiceWithTempBucket(sec.COS.BucketName, sec.COS.Region, sec.COS.SecretID, sec.COS.SecretKey, pathPrefix, sec.COS.TempBucketName, sec.COS.TempRegion)
|
||||
return svc, p, err
|
||||
|
||||
case "tos":
|
||||
if sec == nil || sec.TOS == nil || sec.TOS.Endpoint == "" || sec.TOS.Region == "" || sec.TOS.AccessKey == "" || sec.TOS.SecretKey == "" || sec.TOS.BucketName == "" {
|
||||
return nil, p, fmt.Errorf("incomplete tos config")
|
||||
}
|
||||
svc, err := NewTosFileService(sec.TOS.Endpoint, sec.TOS.Region, sec.TOS.AccessKey, sec.TOS.SecretKey, sec.TOS.BucketName, sec.TOS.PathPrefix)
|
||||
svc, err := NewTosFileServiceWithTempBucket(sec.TOS.Endpoint, sec.TOS.Region, sec.TOS.AccessKey, sec.TOS.SecretKey, sec.TOS.BucketName, sec.TOS.PathPrefix, sec.TOS.TempBucketName, sec.TOS.TempRegion)
|
||||
return svc, p, err
|
||||
case "s3":
|
||||
if sec == nil || sec.S3 == nil || sec.S3.Endpoint == "" || sec.S3.Region == "" || sec.S3.AccessKey == "" || sec.S3.SecretKey == "" || sec.S3.BucketName == "" {
|
||||
@@ -99,16 +99,38 @@ func NewFileServiceFromStorageConfig(
|
||||
if pathPrefix == "" {
|
||||
pathPrefix = "weknora/"
|
||||
}
|
||||
svc, err := NewS3FileService(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix)
|
||||
svc, err := NewS3FileServiceWithOptions(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix, sec.S3.ForcePathStyle)
|
||||
return svc, p, err
|
||||
|
||||
case "obs":
|
||||
obsEndpoint := strings.TrimSpace(os.Getenv("OBS_ENDPOINT"))
|
||||
obsRegion := strings.TrimSpace(os.Getenv("OBS_REGION"))
|
||||
obsAccessKey := strings.TrimSpace(os.Getenv("OBS_ACCESS_KEY"))
|
||||
obsSecretKey := strings.TrimSpace(os.Getenv("OBS_SECRET_KEY"))
|
||||
obsBucketName := strings.TrimSpace(os.Getenv("OBS_BUCKET_NAME"))
|
||||
obsPathPrefix := strings.TrimSpace(os.Getenv("OBS_PATH_PREFIX"))
|
||||
obsEndpoint, obsRegion, obsAccessKey := "", "", ""
|
||||
obsSecretKey, obsBucketName, obsPathPrefix := "", "", ""
|
||||
if sec != nil && sec.OBS != nil {
|
||||
obsEndpoint = strings.TrimSpace(sec.OBS.Endpoint)
|
||||
obsRegion = strings.TrimSpace(sec.OBS.Region)
|
||||
obsAccessKey = strings.TrimSpace(sec.OBS.AccessKey)
|
||||
obsSecretKey = strings.TrimSpace(sec.OBS.SecretKey)
|
||||
obsBucketName = strings.TrimSpace(sec.OBS.BucketName)
|
||||
obsPathPrefix = strings.TrimSpace(sec.OBS.PathPrefix)
|
||||
}
|
||||
if obsEndpoint == "" {
|
||||
obsEndpoint = strings.TrimSpace(os.Getenv("OBS_ENDPOINT"))
|
||||
}
|
||||
if obsRegion == "" {
|
||||
obsRegion = strings.TrimSpace(os.Getenv("OBS_REGION"))
|
||||
}
|
||||
if obsAccessKey == "" {
|
||||
obsAccessKey = strings.TrimSpace(os.Getenv("OBS_ACCESS_KEY"))
|
||||
}
|
||||
if obsSecretKey == "" {
|
||||
obsSecretKey = strings.TrimSpace(os.Getenv("OBS_SECRET_KEY"))
|
||||
}
|
||||
if obsBucketName == "" {
|
||||
obsBucketName = strings.TrimSpace(os.Getenv("OBS_BUCKET_NAME"))
|
||||
}
|
||||
if obsPathPrefix == "" {
|
||||
obsPathPrefix = strings.TrimSpace(os.Getenv("OBS_PATH_PREFIX"))
|
||||
}
|
||||
if obsPathPrefix == "" {
|
||||
obsPathPrefix = "weknora/"
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ type s3FileService struct {
|
||||
}
|
||||
|
||||
// newS3Client creates a bare s3FileService with just the SDK client initialised.
|
||||
func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string) (*s3FileService, error) {
|
||||
func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (*s3FileService, error) {
|
||||
var cfg aws.Config
|
||||
var err error
|
||||
|
||||
@@ -49,7 +49,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
|
||||
// (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
|
||||
var client *s3.Client
|
||||
if endpoint != "" {
|
||||
usePathStyle := !strings.Contains(endpoint, "amazonaws.com")
|
||||
usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")
|
||||
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
o.UsePathStyle = usePathStyle
|
||||
@@ -76,7 +76,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
|
||||
func NewS3FileService(endpoint,
|
||||
accessKey, secretKey, bucketName, region, pathPrefix string,
|
||||
) (interfaces.FileService, error) {
|
||||
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix)
|
||||
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -96,6 +96,25 @@ func NewS3FileService(endpoint,
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// NewS3FileServiceWithOptions is the instance-aware S3 constructor. Existing
|
||||
// callers keep the historical endpoint-based path-style inference.
|
||||
func NewS3FileServiceWithOptions(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (interfaces.FileService, error) {
|
||||
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix, forcePathStyle)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exists, err := svc.bucketExists(context.Background())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err = svc.createBucket(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
}
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// bucketExists checks if the bucket exists
|
||||
func (s *s3FileService) bucketExists(ctx context.Context) (bool, error) {
|
||||
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
|
||||
@@ -145,7 +164,11 @@ func (s *s3FileService) CheckConnectivity(ctx context.Context) error {
|
||||
// CheckS3Connectivity tests S3 connectivity using the provided credentials.
|
||||
// It creates a temporary service instance internally and delegates to CheckConnectivity.
|
||||
func CheckS3Connectivity(ctx context.Context, endpoint, accessKey, secretKey, bucketName, region string) error {
|
||||
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, "")
|
||||
return CheckS3ConnectivityWithOptions(ctx, endpoint, accessKey, secretKey, bucketName, region, false)
|
||||
}
|
||||
|
||||
func CheckS3ConnectivityWithOptions(ctx context.Context, endpoint, accessKey, secretKey, bucketName, region string, forcePathStyle bool) error {
|
||||
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, "", forcePathStyle)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
@@ -79,7 +78,8 @@ type ImageMultimodalService struct {
|
||||
// (e.g. images were saved using the global MINIO_* env vars while the
|
||||
// tenant's StorageEngineConfig.MinIO is empty). Mirrors the write-side
|
||||
// fallback in knowledgeService.resolveFileService.
|
||||
fileSvc interfaces.FileService
|
||||
fileSvc interfaces.FileService
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
|
||||
// spanTracker records this image's subspan under the parent attempt's
|
||||
// multimodal stage. nil-safe — falls back to no-op via tracker().
|
||||
@@ -98,21 +98,23 @@ func NewImageMultimodalService(
|
||||
taskEnqueuer interfaces.TaskEnqueuer,
|
||||
redisClient *redis.Client,
|
||||
fileSvc interfaces.FileService,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
spanTracker SpanTracker,
|
||||
) interfaces.TaskHandler {
|
||||
return &ImageMultimodalService{
|
||||
chunkService: chunkService,
|
||||
modelService: modelService,
|
||||
kbService: kbService,
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
tenantRepo: tenantRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
ollamaService: ollamaService,
|
||||
taskEnqueuer: taskEnqueuer,
|
||||
redisClient: redisClient,
|
||||
fileSvc: fileSvc,
|
||||
spanTracker: spanTracker,
|
||||
chunkService: chunkService,
|
||||
modelService: modelService,
|
||||
kbService: kbService,
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
tenantRepo: tenantRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
ollamaService: ollamaService,
|
||||
taskEnqueuer: taskEnqueuer,
|
||||
redisClient: redisClient,
|
||||
fileSvc: fileSvc,
|
||||
storageResolver: storageResolver,
|
||||
spanTracker: spanTracker,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -550,6 +552,7 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
|
||||
return s.fileSvc
|
||||
}
|
||||
|
||||
backendID, _, _ := types.ParseStorageBackendPath(payload.ImageURL)
|
||||
provider := types.ParseProviderScheme(payload.ImageURL)
|
||||
if provider == "" {
|
||||
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
@@ -557,13 +560,20 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
|
||||
logger.Warnf(ctx, "[ImageMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr)
|
||||
} else if kb != nil {
|
||||
provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider()))
|
||||
if backendID == "" && kb.StorageBackendID != nil {
|
||||
backendID = *kb.StorageBackendID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if s.storageResolver == nil {
|
||||
return s.fileSvc
|
||||
}
|
||||
|
||||
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
||||
logger.Infof(ctx, "[ImageMultimodal] resolving file service: tenant=%d provider=%q LOCAL_STORAGE_BASE_DIR=%q imageURL=%s",
|
||||
payload.TenantID, provider, baseDir, payload.ImageURL)
|
||||
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
|
||||
fileSvc, _, svcErr := s.storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir)
|
||||
if svcErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] resolve file service failed (falling back to default): tenant=%d provider=%s err=%v",
|
||||
payload.TenantID, provider, svcErr)
|
||||
|
||||
@@ -55,6 +55,7 @@ type knowledgeService struct {
|
||||
tagRepo interfaces.KnowledgeTagRepository
|
||||
tagService interfaces.KnowledgeTagService
|
||||
fileSvc interfaces.FileService
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
modelService interfaces.ModelService
|
||||
task interfaces.TaskEnqueuer
|
||||
taskInspector interfaces.TaskInspector
|
||||
@@ -96,6 +97,7 @@ func NewKnowledgeService(
|
||||
tagRepo interfaces.KnowledgeTagRepository,
|
||||
tagService interfaces.KnowledgeTagService,
|
||||
fileSvc interfaces.FileService,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
modelService interfaces.ModelService,
|
||||
task interfaces.TaskEnqueuer,
|
||||
taskInspector interfaces.TaskInspector,
|
||||
@@ -122,6 +124,7 @@ func NewKnowledgeService(
|
||||
tagRepo: tagRepo,
|
||||
tagService: tagService,
|
||||
fileSvc: fileSvc,
|
||||
storageResolver: storageResolver,
|
||||
modelService: modelService,
|
||||
task: task,
|
||||
taskInspector: taskInspector,
|
||||
|
||||
@@ -142,15 +142,18 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
|
||||
}
|
||||
|
||||
switch provider {
|
||||
case "cos":
|
||||
concreteBackend := kb.StorageBackendID != nil && strings.TrimSpace(*kb.StorageBackendID) != ""
|
||||
switch {
|
||||
case concreteBackend:
|
||||
// Registration already validated and tested the concrete instance.
|
||||
case provider == "cos":
|
||||
if tenant == nil || tenant.StorageEngineConfig == nil || tenant.StorageEngineConfig.COS == nil ||
|
||||
tenant.StorageEngineConfig.COS.SecretID == "" || tenant.StorageEngineConfig.COS.SecretKey == "" ||
|
||||
tenant.StorageEngineConfig.COS.Region == "" || tenant.StorageEngineConfig.COS.BucketName == "" {
|
||||
logger.Error(ctx, "COS configuration incomplete for image multimodal processing")
|
||||
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
|
||||
}
|
||||
case "minio":
|
||||
case provider == "minio":
|
||||
ok := false
|
||||
if tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
|
||||
m := tenant.StorageEngineConfig.MinIO
|
||||
|
||||
@@ -235,6 +235,11 @@ func mergeExtractConfig(base types.ExtractConfig, override *types.ExtractConfig)
|
||||
}
|
||||
|
||||
func validateImageMultimodalConfig(ctx context.Context, kb *types.KnowledgeBase) error {
|
||||
// Concrete backends are validated and connectivity-tested when registered.
|
||||
// The checks below only apply to unmigrated provider-only bindings.
|
||||
if kb != nil && kb.StorageBackendID != nil && strings.TrimSpace(*kb.StorageBackendID) != "" {
|
||||
return nil
|
||||
}
|
||||
provider := kb.GetStorageProvider()
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
|
||||
@@ -115,6 +115,19 @@ func (s *knowledgeService) getVLMConfig(ctx context.Context, kb *types.Knowledge
|
||||
|
||||
func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.KnowledgeBase) *types.DocParserStorageConfig {
|
||||
provider := kb.GetStorageProvider()
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
backendID := ""
|
||||
if kb.StorageBackendID != nil {
|
||||
backendID = *kb.StorageBackendID
|
||||
}
|
||||
if s.storageResolver != nil && tenant != nil {
|
||||
if backend, err := s.storageResolver.ResolveBackend(ctx, tenant, backendID, provider); err == nil && backend != nil {
|
||||
provider = backend.Provider
|
||||
tenantCopy := *tenant
|
||||
tenantCopy.StorageEngineConfig = backend.ToStorageEngineConfig()
|
||||
tenant = &tenantCopy
|
||||
}
|
||||
}
|
||||
if provider == "" {
|
||||
provider = "local"
|
||||
}
|
||||
@@ -131,7 +144,7 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
|
||||
hasKBFull = sc.SecretID != "" && sc.BucketName != ""
|
||||
case "minio":
|
||||
hasKBFull = sc.BucketName != ""
|
||||
case "local", "tos", "s3", "oss", "ks3":
|
||||
case "local", "tos", "s3", "oss", "ks3", "obs":
|
||||
hasKBFull = false
|
||||
}
|
||||
|
||||
@@ -153,7 +166,6 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
|
||||
var out types.DocParserStorageConfig
|
||||
out.Provider = strings.ToUpper(provider)
|
||||
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
sec := tenant.StorageEngineConfig
|
||||
if sec.DefaultProvider != "" && provider == "" {
|
||||
@@ -228,6 +240,15 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
|
||||
out.BucketName = sec.KS3.BucketName
|
||||
out.PathPrefix = sec.KS3.PathPrefix
|
||||
}
|
||||
case "obs":
|
||||
if sec.OBS != nil {
|
||||
out.Endpoint = sec.OBS.Endpoint
|
||||
out.Region = sec.OBS.Region
|
||||
out.AccessKeyID = sec.OBS.AccessKey
|
||||
out.SecretAccessKey = sec.OBS.SecretKey
|
||||
out.BucketName = sec.OBS.BucketName
|
||||
out.PathPrefix = sec.OBS.PathPrefix
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -248,6 +269,21 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
|
||||
provider := kb.GetStorageProvider()
|
||||
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
backendID := ""
|
||||
if kb.StorageBackendID != nil {
|
||||
backendID = strings.TrimSpace(*kb.StorageBackendID)
|
||||
}
|
||||
if s.storageResolver != nil && tenant != nil {
|
||||
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
||||
svc, resolvedProvider, err := s.storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir)
|
||||
if err == nil && svc != nil {
|
||||
logger.Infof(ctx, "[storage] resolveFileService selected instance: kb=%s backend=%s provider=%s", kb.ID, backendID, resolvedProvider)
|
||||
return svc
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to resolve storage backend for kb=%s: %v", kb.ID, err)
|
||||
}
|
||||
}
|
||||
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
|
||||
}
|
||||
@@ -274,6 +310,18 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
|
||||
// the provider inferred from the file path. This protects historical data when
|
||||
// tenant/KB config changes but files were stored under the old provider.
|
||||
func (s *knowledgeService) resolveFileServiceForPath(ctx context.Context, kb *types.KnowledgeBase, filePath string) interfaces.FileService {
|
||||
if backendID, inner, ok := types.ParseStorageBackendPath(filePath); ok && s.storageResolver != nil {
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant != nil {
|
||||
provider := types.ParseProviderScheme(inner)
|
||||
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
||||
if resolved, _, err := s.storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir); err == nil {
|
||||
return resolved
|
||||
} else {
|
||||
logger.Warnf(ctx, "[storage] failed to resolve backend from file path: backend=%s err=%v", backendID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
svc := s.resolveFileService(ctx, kb)
|
||||
if filePath == "" {
|
||||
return svc
|
||||
|
||||
@@ -27,21 +27,22 @@ var ErrInvalidTenantID = errors.New("invalid tenant ID")
|
||||
|
||||
// knowledgeBaseService implements the knowledge base service interface
|
||||
type knowledgeBaseService struct {
|
||||
repo interfaces.KnowledgeBaseRepository
|
||||
kgRepo interfaces.KnowledgeRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
shareRepo interfaces.KBShareRepository
|
||||
kbShareService interfaces.KBShareService
|
||||
modelService interfaces.ModelService
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ownership retriever.TenantStoreOwnership
|
||||
tenantRepo interfaces.TenantRepository
|
||||
fileSvc interfaces.FileService
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
asynqClient interfaces.TaskEnqueuer
|
||||
dsRepo interfaces.DataSourceRepository
|
||||
syncLogRepo interfaces.SyncLogRepository
|
||||
dsScheduler *datasource.Scheduler
|
||||
repo interfaces.KnowledgeBaseRepository
|
||||
kgRepo interfaces.KnowledgeRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
shareRepo interfaces.KBShareRepository
|
||||
kbShareService interfaces.KBShareService
|
||||
modelService interfaces.ModelService
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ownership retriever.TenantStoreOwnership
|
||||
tenantRepo interfaces.TenantRepository
|
||||
fileSvc interfaces.FileService
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
asynqClient interfaces.TaskEnqueuer
|
||||
dsRepo interfaces.DataSourceRepository
|
||||
syncLogRepo interfaces.SyncLogRepository
|
||||
dsScheduler *datasource.Scheduler
|
||||
}
|
||||
|
||||
// NewKnowledgeBaseService creates a new knowledge base service
|
||||
@@ -55,6 +56,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
|
||||
ownership retriever.TenantStoreOwnership,
|
||||
tenantRepo interfaces.TenantRepository,
|
||||
fileSvc interfaces.FileService,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
asynqClient interfaces.TaskEnqueuer,
|
||||
dsRepo interfaces.DataSourceRepository,
|
||||
@@ -62,21 +64,22 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
|
||||
dsScheduler *datasource.Scheduler,
|
||||
) interfaces.KnowledgeBaseService {
|
||||
return &knowledgeBaseService{
|
||||
repo: repo,
|
||||
kgRepo: kgRepo,
|
||||
chunkRepo: chunkRepo,
|
||||
shareRepo: shareRepo,
|
||||
kbShareService: kbShareService,
|
||||
modelService: modelService,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
tenantRepo: tenantRepo,
|
||||
fileSvc: fileSvc,
|
||||
graphEngine: graphEngine,
|
||||
asynqClient: asynqClient,
|
||||
dsRepo: dsRepo,
|
||||
syncLogRepo: syncLogRepo,
|
||||
dsScheduler: dsScheduler,
|
||||
repo: repo,
|
||||
kgRepo: kgRepo,
|
||||
chunkRepo: chunkRepo,
|
||||
shareRepo: shareRepo,
|
||||
kbShareService: kbShareService,
|
||||
modelService: modelService,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
tenantRepo: tenantRepo,
|
||||
fileSvc: fileSvc,
|
||||
storageResolver: storageResolver,
|
||||
graphEngine: graphEngine,
|
||||
asynqClient: asynqClient,
|
||||
dsRepo: dsRepo,
|
||||
syncLogRepo: syncLogRepo,
|
||||
dsScheduler: dsScheduler,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,6 +122,9 @@ func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
|
||||
}
|
||||
kb.EnsureDefaults()
|
||||
applyTenantDefaultStorageProvider(ctx, kb)
|
||||
if err := s.applyAndValidateStorageBackend(ctx, kb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Fold empty-string vector_store_id into nil so this path and the
|
||||
// retrieve-engine factory's pre-condition share a single representation.
|
||||
@@ -150,6 +156,37 @@ func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
|
||||
return kb, nil
|
||||
}
|
||||
|
||||
func (s *knowledgeBaseService) applyAndValidateStorageBackend(ctx context.Context, kb *types.KnowledgeBase) error {
|
||||
if s.storageResolver == nil || kb == nil {
|
||||
return nil
|
||||
}
|
||||
tenant, _ := types.TenantInfoFromContext(ctx)
|
||||
if tenant == nil {
|
||||
return apperrors.NewBadRequestError("workspace context missing")
|
||||
}
|
||||
id := ""
|
||||
if kb.StorageBackendID != nil {
|
||||
id = strings.TrimSpace(*kb.StorageBackendID)
|
||||
}
|
||||
provider := kb.GetStorageProvider()
|
||||
// A newly created KB without an explicit instance follows the concrete
|
||||
// tenant default. The legacy provider is only a fallback for workspaces
|
||||
// that have not been migrated yet.
|
||||
if id == "" && tenant.DefaultStorageBackendID != nil && strings.TrimSpace(*tenant.DefaultStorageBackendID) != "" {
|
||||
provider = ""
|
||||
}
|
||||
backend, err := s.storageResolver.ResolveBackend(ctx, tenant, id, provider)
|
||||
if err != nil {
|
||||
return apperrors.NewBadRequestError("storage backend is unavailable").WithDetails(err.Error())
|
||||
}
|
||||
if backend == nil {
|
||||
return nil
|
||||
}
|
||||
kb.StorageBackendID = &backend.ID
|
||||
kb.SetStorageProvider(backend.Provider)
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyTenantDefaultStorageProvider fills an empty KB storage provider from the
|
||||
// tenant's global default (Settings → Storage engine). Frontend should send the
|
||||
// same value; this keeps API clients and legacy UIs consistent.
|
||||
@@ -978,22 +1015,19 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context,
|
||||
"cross-store cloning is not yet supported")
|
||||
}
|
||||
|
||||
// Defense 3: storage backend must match — only meaningful when the
|
||||
// tenant has a StorageEngineConfig. Without it, resolveFileService
|
||||
// ignores per-KB provider pins and routes ALL KBs to the global
|
||||
// storage service, so a clone can never span two real backends and
|
||||
// the pins must NOT be used to reject (that would be a false positive).
|
||||
// When a tenant config exists, pins are honored, so compare effective
|
||||
// providers and reject a genuine cross-backend clone up front (it would
|
||||
// otherwise fail mid-clone with ErrCrossBackendCopy).
|
||||
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
tenantDefault := tenant.StorageEngineConfig.DefaultProvider
|
||||
srcProvider := sourceKB.EffectiveStorageProvider(tenantDefault)
|
||||
dstProvider := targetKB.EffectiveStorageProvider(tenantDefault)
|
||||
if srcProvider != "" && dstProvider != "" && srcProvider != dstProvider {
|
||||
return nil, nil, apperrors.NewBadRequestError(fmt.Sprintf(
|
||||
"source and target knowledge bases use different storage backends (%s vs %s); "+
|
||||
"cross-storage-backend cloning is not supported", srcProvider, dstProvider))
|
||||
// Defense 3: the concrete storage instance must match. Comparing only
|
||||
// provider names would incorrectly allow COS-A -> COS-B clones.
|
||||
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil {
|
||||
defaultID, defaultProvider := "", ""
|
||||
if tenant.DefaultStorageBackendID != nil {
|
||||
defaultID = *tenant.DefaultStorageBackendID
|
||||
}
|
||||
if tenant.StorageEngineConfig != nil {
|
||||
defaultProvider = tenant.StorageEngineConfig.DefaultProvider
|
||||
}
|
||||
if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) {
|
||||
return nil, nil, apperrors.NewBadRequestError(
|
||||
"source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1016,6 +1050,7 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context,
|
||||
SummaryModelID: sourceKB.SummaryModelID,
|
||||
VLMConfig: sourceKB.VLMConfig,
|
||||
StorageProviderConfig: sourceKB.StorageProviderConfig,
|
||||
StorageBackendID: sourceKB.StorageBackendID,
|
||||
StorageConfig: sourceKB.StorageConfig,
|
||||
FAQConfig: faqConfig,
|
||||
VectorStoreID: sourceKB.VectorStoreID,
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
type StorageBackendService struct {
|
||||
repo interfaces.StorageBackendRepository
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func NewStorageBackendService(repo interfaces.StorageBackendRepository, db *gorm.DB) *StorageBackendService {
|
||||
return &StorageBackendService{repo: repo, db: db}
|
||||
}
|
||||
|
||||
func (s *StorageBackendService) Create(ctx context.Context, backend *types.StorageBackend) error {
|
||||
if err := backend.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateStorageBackendEndpoint(backend); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Test(ctx, backend); err != nil {
|
||||
return apperrors.NewBadRequestError("storage connection test failed").WithDetails(err.Error())
|
||||
}
|
||||
backend.CreatedAt, backend.UpdatedAt = time.Now(), time.Now()
|
||||
if err := s.repo.Create(ctx, backend); err != nil {
|
||||
if strings.Contains(strings.ToLower(err.Error()), "unique") {
|
||||
return apperrors.NewConflictError("a storage backend with this name already exists")
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *StorageBackendService) Update(ctx context.Context, incoming *types.StorageBackend) error {
|
||||
existing, err := s.repo.GetByID(ctx, incoming.TenantID, incoming.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing == nil {
|
||||
return apperrors.NewNotFoundError("storage backend not found")
|
||||
}
|
||||
if existing.Source == types.StorageBackendSourceEnv {
|
||||
return apperrors.NewBadRequestError("environment storage backend is read-only")
|
||||
}
|
||||
incoming.Provider = existing.Provider
|
||||
incoming.Config = incoming.Config.MergeSecrets(existing.Config)
|
||||
if incoming.Config.LocationKey(existing.Provider) != existing.Config.LocationKey(existing.Provider) {
|
||||
return apperrors.NewBadRequestError("endpoint, region, bucket and path prefix are immutable; use storage migration instead")
|
||||
}
|
||||
if incoming.Status == "" {
|
||||
incoming.Status = existing.Status
|
||||
}
|
||||
if incoming.Status == types.StorageBackendStatusDisabled && existing.Status != types.StorageBackendStatusDisabled {
|
||||
var references int64
|
||||
if err := s.db.WithContext(ctx).Model(&types.Tenant{}).Where("id = ? AND default_storage_backend_id = ?", incoming.TenantID, incoming.ID).Count(&references).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if references == 0 {
|
||||
if err := s.db.WithContext(ctx).Model(&types.KnowledgeBase{}).Where("tenant_id = ? AND storage_backend_id = ?", incoming.TenantID, incoming.ID).Count(&references).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if references > 0 {
|
||||
return apperrors.NewBadRequestError("a default or bound storage backend cannot be disabled")
|
||||
}
|
||||
}
|
||||
if err := incoming.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateStorageBackendEndpoint(incoming); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Test(ctx, incoming); err != nil {
|
||||
return apperrors.NewBadRequestError("storage connection test failed").WithDetails(err.Error())
|
||||
}
|
||||
incoming.UpdatedAt = time.Now()
|
||||
return s.repo.Update(ctx, incoming)
|
||||
}
|
||||
|
||||
func (s *StorageBackendService) Delete(ctx context.Context, tenantID uint64, id string) error {
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var backend types.StorageBackend
|
||||
query := tx.Where("tenant_id = ? AND id = ?", tenantID, id)
|
||||
if tx.Dialector.Name() == "postgres" {
|
||||
query = query.Clauses(clause.Locking{Strength: "UPDATE"})
|
||||
}
|
||||
if err := query.First(&backend).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return apperrors.NewNotFoundError("storage backend not found")
|
||||
}
|
||||
return err
|
||||
}
|
||||
if backend.Source == types.StorageBackendSourceEnv {
|
||||
return apperrors.NewBadRequestError("environment storage backend is read-only")
|
||||
}
|
||||
var defaultCount int64
|
||||
if err := tx.Model(&types.Tenant{}).Where("id = ? AND default_storage_backend_id = ?", tenantID, id).Count(&defaultCount).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if defaultCount > 0 {
|
||||
return apperrors.NewBadRequestError("default storage backend cannot be deleted")
|
||||
}
|
||||
var kbCount int64
|
||||
if err := tx.Model(&types.KnowledgeBase{}).Where("tenant_id = ? AND storage_backend_id = ?", tenantID, id).Count(&kbCount).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if kbCount > 0 {
|
||||
return apperrors.NewBadRequestError(fmt.Sprintf("storage backend still has %d knowledge base(s) bound to it", kbCount))
|
||||
}
|
||||
if backend.LegacyAlias {
|
||||
return apperrors.NewBadRequestError("legacy storage backend cannot be deleted while old file paths may reference it")
|
||||
}
|
||||
return tx.Delete(&backend).Error
|
||||
})
|
||||
}
|
||||
|
||||
func (s *StorageBackendService) SetDefault(ctx context.Context, tenantID uint64, id string) error {
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var backend types.StorageBackend
|
||||
query := tx.Where("tenant_id = ? AND id = ?", tenantID, id)
|
||||
if tx.Dialector.Name() == "postgres" {
|
||||
query = query.Clauses(clause.Locking{Strength: "UPDATE"})
|
||||
}
|
||||
if err := query.First(&backend).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return apperrors.NewNotFoundError("storage backend not found")
|
||||
}
|
||||
return err
|
||||
}
|
||||
if backend.Status != types.StorageBackendStatusActive {
|
||||
return apperrors.NewBadRequestError("only an active storage backend can be the default")
|
||||
}
|
||||
return tx.Model(&types.Tenant{}).Where("id = ?", tenantID).Update("default_storage_backend_id", id).Error
|
||||
})
|
||||
}
|
||||
|
||||
func (s *StorageBackendService) Test(ctx context.Context, backend *types.StorageBackend) error {
|
||||
if err := backend.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateStorageBackendEndpoint(backend); err != nil {
|
||||
return err
|
||||
}
|
||||
if backend.Provider == "local" {
|
||||
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
||||
if baseDir == "" {
|
||||
baseDir = "/data/files"
|
||||
}
|
||||
candidate := filepath.Join(baseDir, strings.Trim(strings.TrimSpace(backend.Config.PathPrefix), "/\\"))
|
||||
safeDir, err := secutils.SafePathUnderBase(baseDir, candidate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(safeDir, 0o755); err != nil {
|
||||
return fmt.Errorf("create local storage directory: %w", err)
|
||||
}
|
||||
}
|
||||
c := backend.Config
|
||||
switch backend.Provider {
|
||||
case "local":
|
||||
fileService, _, err := filesvc.NewFileServiceFromStorageConfig("local", backend.ToStorageEngineConfig(), "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fileService.CheckConnectivity(ctx)
|
||||
case "minio":
|
||||
if c.Mode == "docker" {
|
||||
c.Endpoint = os.Getenv("MINIO_ENDPOINT")
|
||||
c.AccessKeyID = os.Getenv("MINIO_ACCESS_KEY_ID")
|
||||
c.SecretAccessKey = os.Getenv("MINIO_SECRET_ACCESS_KEY")
|
||||
if c.BucketName == "" {
|
||||
c.BucketName = os.Getenv("MINIO_BUCKET_NAME")
|
||||
}
|
||||
}
|
||||
return filesvc.CheckMinioConnectivity(ctx, c.Endpoint, c.AccessKeyID, c.SecretAccessKey, c.BucketName, c.UseSSL)
|
||||
case "cos":
|
||||
return filesvc.CheckCosConnectivity(ctx, c.BucketName, c.Region, c.AccessKeyID, c.SecretAccessKey)
|
||||
case "tos":
|
||||
return filesvc.CheckTosConnectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
|
||||
case "s3":
|
||||
return filesvc.CheckS3ConnectivityWithOptions(ctx, c.Endpoint, c.AccessKeyID, c.SecretAccessKey, c.BucketName, c.Region, c.ForcePathStyle)
|
||||
case "oss":
|
||||
return filesvc.CheckOssConnectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
|
||||
case "ks3":
|
||||
return filesvc.CheckKS3Connectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
|
||||
case "obs":
|
||||
return filesvc.CheckObsConnectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
|
||||
default:
|
||||
return fmt.Errorf("unsupported storage provider: %s", backend.Provider)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StorageBackendService) ResolveBackend(ctx context.Context, tenant *types.Tenant, backendID, provider string) (*types.StorageBackend, error) {
|
||||
if tenant == nil {
|
||||
return nil, fmt.Errorf("workspace context missing")
|
||||
}
|
||||
backendID = strings.TrimSpace(backendID)
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
if backendID == "" && provider != "" {
|
||||
backend, err := s.repo.FindLegacyAlias(ctx, tenant.ID, provider)
|
||||
if err != nil || backend != nil {
|
||||
return backend, err
|
||||
}
|
||||
}
|
||||
if backendID == "" && tenant.DefaultStorageBackendID != nil {
|
||||
backendID = strings.TrimSpace(*tenant.DefaultStorageBackendID)
|
||||
}
|
||||
if backendID != "" {
|
||||
backend, err := s.repo.GetByID(ctx, tenant.ID, backendID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if backend == nil {
|
||||
return nil, fmt.Errorf("storage backend not found")
|
||||
}
|
||||
if backend.Status != types.StorageBackendStatusActive {
|
||||
return nil, fmt.Errorf("storage backend is not active")
|
||||
}
|
||||
return backend, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *StorageBackendService) ResolveFileService(ctx context.Context, tenant *types.Tenant, backendID, provider, localBaseDir string) (interfaces.FileService, string, error) {
|
||||
backend, err := s.ResolveBackend(ctx, tenant, backendID, provider)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if backend != nil {
|
||||
inner, provider, err := filesvc.NewFileServiceFromStorageConfig(backend.Provider, backend.ToStorageEngineConfig(), localBaseDir)
|
||||
if err != nil {
|
||||
return nil, provider, err
|
||||
}
|
||||
return filesvc.NewBackendScopedFileService(backend.ID, inner), provider, nil
|
||||
}
|
||||
if tenant == nil {
|
||||
return nil, "", fmt.Errorf("workspace context missing")
|
||||
}
|
||||
return filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, localBaseDir)
|
||||
}
|
||||
|
||||
func validateStorageBackendEndpoint(backend *types.StorageBackend) error {
|
||||
if backend.Provider == "local" || (backend.Provider == "minio" && backend.Config.Mode == "docker") {
|
||||
return nil
|
||||
}
|
||||
endpoint := strings.TrimSpace(backend.Config.Endpoint)
|
||||
if backend.Provider == "cos" || endpoint == "" {
|
||||
return nil
|
||||
}
|
||||
if !strings.Contains(endpoint, "://") {
|
||||
scheme := "https://"
|
||||
if backend.Provider == "minio" && !backend.Config.UseSSL {
|
||||
scheme = "http://"
|
||||
}
|
||||
endpoint = scheme + endpoint
|
||||
}
|
||||
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return apperrors.NewBadRequestError("storage endpoint failed SSRF validation").WithDetails(err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ interfaces.StorageBackendService = (*StorageBackendService)(nil)
|
||||
var _ interfaces.StorageBackendResolver = (*StorageBackendService)(nil)
|
||||
@@ -0,0 +1,42 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestStorageBackendResolverScopesPathsAndTenant(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.AutoMigrate(&types.StorageBackend{}))
|
||||
repo := repository.NewStorageBackendRepository(db)
|
||||
backend := &types.StorageBackend{TenantID: 7, Name: "Local A", Provider: "local", Config: types.StorageBackendConfig{}, LegacyAlias: true}
|
||||
require.NoError(t, repo.Create(context.Background(), backend))
|
||||
|
||||
resolver := service.NewStorageBackendService(repo, db)
|
||||
svc, provider, err := resolver.ResolveFileService(context.Background(), &types.Tenant{ID: 7}, backend.ID, "local", t.TempDir())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "local", provider)
|
||||
|
||||
path, err := svc.SaveBytes(context.Background(), []byte("scoped"), 7, "exports/a.txt", false)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, path, "storage://"+backend.ID+"/local://")
|
||||
reader, err := svc.GetFile(context.Background(), path)
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
data, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "scoped", string(data))
|
||||
|
||||
_, _, err = resolver.ResolveFileService(context.Background(), &types.Tenant{ID: 8}, backend.ID, "local", t.TempDir())
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -21,12 +21,13 @@ type ListTenantsParams struct {
|
||||
|
||||
// tenantService implements the TenantService interface
|
||||
type tenantService struct {
|
||||
repo interfaces.TenantRepository // Repository for tenant data operations
|
||||
repo interfaces.TenantRepository // Repository for tenant data operations
|
||||
storageRepo interfaces.StorageBackendRepository
|
||||
}
|
||||
|
||||
// NewTenantService creates a new tenant service instance
|
||||
func NewTenantService(repo interfaces.TenantRepository) interfaces.TenantService {
|
||||
return &tenantService{repo: repo}
|
||||
func NewTenantService(repo interfaces.TenantRepository, storageRepo interfaces.StorageBackendRepository) interfaces.TenantService {
|
||||
return &tenantService{repo: repo, storageRepo: storageRepo}
|
||||
}
|
||||
|
||||
// CreateTenant creates a new tenant
|
||||
@@ -60,11 +61,44 @@ func (s *tenantService) CreateTenant(ctx context.Context, tenant *types.Tenant)
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
if err := s.createDefaultStorageBackend(ctx, tenant); err != nil {
|
||||
// No related rows exist yet, so rolling the tenant back is safe and
|
||||
// avoids leaving a workspace that cannot bind new knowledge bases.
|
||||
_ = s.repo.DeleteTenant(ctx, tenant.ID)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Tenant created successfully, ID: %d, name: %s", tenant.ID, tenant.Name)
|
||||
return tenant, nil
|
||||
}
|
||||
|
||||
func (s *tenantService) createDefaultStorageBackend(ctx context.Context, tenant *types.Tenant) error {
|
||||
if s.storageRepo == nil || tenant == nil {
|
||||
return nil
|
||||
}
|
||||
provider := ""
|
||||
if tenant.StorageEngineConfig != nil {
|
||||
provider = tenant.StorageEngineConfig.DefaultProvider
|
||||
}
|
||||
backend := types.StorageBackendFromLegacy(tenant.ID, provider, tenant.StorageEngineConfig)
|
||||
if backend == nil {
|
||||
backend = types.StorageBackendFromEnvironment(tenant.ID)
|
||||
}
|
||||
if backend == nil {
|
||||
return errors.New("no supported default storage backend is configured")
|
||||
}
|
||||
backend.LegacyAlias = true
|
||||
if err := s.storageRepo.Create(ctx, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
tenant.DefaultStorageBackendID = &backend.ID
|
||||
if err := s.repo.UpdateTenant(ctx, tenant); err != nil {
|
||||
_ = s.storageRepo.Delete(ctx, tenant.ID, backend.ID)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTenantByID retrieves a tenant by their ID
|
||||
func (s *tenantService) GetTenantByID(ctx context.Context, id uint64) (*types.Tenant, error) {
|
||||
if id == 0 {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestCreateTenantCreatesConcreteDefaultStorageBackend(t *testing.T) {
|
||||
t.Setenv("STORAGE_TYPE", "local")
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.AutoMigrate(&types.Tenant{}, &types.StorageBackend{}))
|
||||
tenantRepo := repository.NewTenantRepository(db)
|
||||
storageRepo := repository.NewStorageBackendRepository(db)
|
||||
tenantSvc := service.NewTenantService(tenantRepo, storageRepo)
|
||||
|
||||
tenant, err := tenantSvc.CreateTenant(context.Background(), &types.Tenant{Name: "workspace"})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, tenant.DefaultStorageBackendID)
|
||||
|
||||
backend, err := storageRepo.GetByID(context.Background(), tenant.ID, *tenant.DefaultStorageBackendID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, backend)
|
||||
assert.Equal(t, "local", backend.Provider)
|
||||
assert.Equal(t, types.StorageBackendSourceEnv, backend.Source)
|
||||
assert.True(t, backend.LegacyAlias)
|
||||
}
|
||||
@@ -137,7 +137,7 @@ func mockEngineFactory(err error) interfaces.EngineFactory {
|
||||
// mockEngineService satisfies interfaces.RetrieveEngineService minimally.
|
||||
type mockEngineService struct{}
|
||||
|
||||
func (m *mockEngineService) EngineType() types.RetrieverEngineType { return "mock" }
|
||||
func (m *mockEngineService) EngineType() types.RetrieverEngineType { return "mock" }
|
||||
func (m *mockEngineService) Retrieve(_ context.Context, _ types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -842,7 +842,8 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
|
||||
is_pinned INTEGER NOT NULL DEFAULT 0,
|
||||
pinned_at DATETIME NULL,
|
||||
asr_config TEXT,
|
||||
vector_store_id VARCHAR(36),
|
||||
vector_store_id VARCHAR(36),
|
||||
storage_backend_id VARCHAR(36),
|
||||
wiki_config TEXT,
|
||||
indexing_strategy TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
@@ -1077,7 +1078,7 @@ func TestDeleteStore_Guard_RejectsCount3(t *testing.T) {
|
||||
storeID := "store-A"
|
||||
for i := 0; i < 3; i++ {
|
||||
kb := &types.KnowledgeBase{
|
||||
ID: "kb-multi-" + tenantID2s(1) + "-" + ptrOrEmpty(&storeID) + "-" + string(rune('a'+i)),
|
||||
ID: "kb-multi-" + tenantID2s(1) + "-" + ptrOrEmpty(&storeID) + "-" + string(rune('a'+i)),
|
||||
Name: "kb", TenantID: 1, EmbeddingModelID: "e", SummaryModelID: "s",
|
||||
VectorStoreID: &storeID,
|
||||
}
|
||||
|
||||
@@ -81,6 +81,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/router"
|
||||
"github.com/Tencent/WeKnora/internal/storageallowlist"
|
||||
"github.com/Tencent/WeKnora/internal/stream"
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -229,6 +230,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Invoke(registerWebSearchProviders))
|
||||
must(container.Provide(repository.NewWebSearchProviderRepository))
|
||||
must(container.Provide(repository.NewVectorStoreRepository))
|
||||
must(container.Provide(repository.NewStorageBackendRepository))
|
||||
// TenantStoreOwnership adapter used by the retriever factory functions
|
||||
// to verify that a resolved VectorStore belongs to the caller's tenant.
|
||||
must(container.Provide(retriever.NewVectorStoreRepoOwnership))
|
||||
@@ -245,6 +247,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
return sr, nil
|
||||
}))
|
||||
must(container.Provide(service.NewVectorStoreService))
|
||||
must(container.Provide(service.NewStorageBackendService))
|
||||
must(container.Provide(func(s *service.StorageBackendService) interfaces.StorageBackendService { return s }))
|
||||
must(container.Provide(func(s *service.StorageBackendService) interfaces.StorageBackendResolver { return s }))
|
||||
|
||||
// Agent service layer (requires event bus, web search service)
|
||||
// SessionService is passed as parameter to CreateAgentEngine method when creating AgentService
|
||||
@@ -358,6 +363,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Provide(handler.NewWebSearchHandler))
|
||||
must(container.Provide(handler.NewWebSearchProviderHandler))
|
||||
must(container.Provide(handler.NewVectorStoreHandler))
|
||||
must(container.Provide(handler.NewStorageBackendHandler))
|
||||
must(container.Provide(handler.NewCustomAgentHandler))
|
||||
must(container.Provide(handler.NewUserResourceFavoriteHandler))
|
||||
must(container.Provide(service.NewSkillService))
|
||||
@@ -406,7 +412,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
// disk bytes requires rebuilding the FileService from that tenant's storage
|
||||
// config. The owning tenant is parsed from the URL's first path segment, which
|
||||
// correctly handles cross-tenant shared resources (e.g. shared KB images).
|
||||
func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository) {
|
||||
func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository, storageResolver interfaces.StorageBackendResolver) {
|
||||
chat.LocalImageResolver = func(storageURL string) ([]byte, bool) {
|
||||
tenantID := secutils.ParseTenantIDFromStoragePath(storageURL)
|
||||
if tenantID == 0 {
|
||||
@@ -418,7 +424,12 @@ func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository) {
|
||||
return nil, false
|
||||
}
|
||||
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
||||
fileSvc, _, err := file.NewFileServiceFromStorageConfig("local", tenant.StorageEngineConfig, baseDir)
|
||||
backendID, inner, _ := types.ParseStorageBackendPath(storageURL)
|
||||
provider := types.ParseProviderScheme(inner)
|
||||
if provider == "" {
|
||||
provider = "local"
|
||||
}
|
||||
fileSvc, _, err := storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
@@ -665,6 +676,7 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
||||
// The SQL migration marks KBs that have documents but no provider with "__pending_env__";
|
||||
// we replace that with the actual STORAGE_TYPE from the environment.
|
||||
resolveStorageProviderPending(db)
|
||||
migrateLegacyStorageBackends(db)
|
||||
|
||||
// Post-migration: declarative built-in models from config/builtin_models.yaml (optional).
|
||||
if err := types.LoadBuiltinModelsConfig(context.Background(), db, config.ConfigDir()); err != nil {
|
||||
@@ -723,6 +735,96 @@ func resolveStorageProviderPending(db *gorm.DB) {
|
||||
resetPendingTasks(db)
|
||||
}
|
||||
|
||||
// migrateLegacyStorageBackends backfills the storage_backends table from each
|
||||
// workspace's legacy StorageEngineConfig (or environment defaults) and binds
|
||||
// existing knowledge bases to the resulting backend.
|
||||
//
|
||||
// The table, columns and indexes are created by the SQL migrations
|
||||
// (migrations/versioned/000068 for Postgres, migrations/sqlite/000000_init for
|
||||
// SQLite); this step only handles data that cannot be expressed portably in
|
||||
// SQL: environment snapshots, JSON→config mapping, AES-encrypted credentials,
|
||||
// UUID generation and the per-startup refresh of env-backed aliases.
|
||||
// The migration is idempotent: one legacy_alias row per tenant/provider.
|
||||
func migrateLegacyStorageBackends(db *gorm.DB) {
|
||||
var tenants []*types.Tenant
|
||||
if err := db.Find(&tenants).Error; err != nil {
|
||||
logger.Warnf(context.Background(), "Failed to load workspaces for storage backend migration: %v", err)
|
||||
return
|
||||
}
|
||||
for _, tenant := range tenants {
|
||||
legacy := tenant.StorageEngineConfig
|
||||
defaultProvider := ""
|
||||
if legacy != nil {
|
||||
defaultProvider = strings.ToLower(strings.TrimSpace(legacy.DefaultProvider))
|
||||
}
|
||||
if defaultProvider == "" {
|
||||
defaultProvider = strings.ToLower(strings.TrimSpace(os.Getenv("STORAGE_TYPE")))
|
||||
}
|
||||
if defaultProvider == "" {
|
||||
defaultProvider = "local"
|
||||
}
|
||||
|
||||
backendIDs := make(map[string]string)
|
||||
for _, provider := range storageallowlist.Supported() {
|
||||
var existing types.StorageBackend
|
||||
err := db.Where("tenant_id = ? AND provider = ? AND legacy_alias = ?", tenant.ID, provider, true).First(&existing).Error
|
||||
if err == nil {
|
||||
// Environment-backed aliases are snapshots, not user-owned config.
|
||||
// Refresh them at every startup so credential rotation does not
|
||||
// leave the persisted resolver on stale values. If the workspace
|
||||
// later gains an explicit legacy config, promote the alias to user
|
||||
// source and stop automatic refreshes.
|
||||
if existing.Source == types.StorageBackendSourceEnv {
|
||||
desired := types.StorageBackendFromLegacy(tenant.ID, provider, legacy)
|
||||
if desired == nil && provider == defaultProvider {
|
||||
desired = types.StorageBackendFromEnvironment(tenant.ID)
|
||||
}
|
||||
if desired != nil && desired.Provider == provider {
|
||||
_ = db.Model(&types.StorageBackend{}).Where("id = ?", existing.ID).Updates(map[string]interface{}{
|
||||
"name": desired.Name, "config": desired.Config, "source": desired.Source, "status": desired.Status, "updated_at": time.Now(),
|
||||
}).Error
|
||||
}
|
||||
}
|
||||
backendIDs[provider] = existing.ID
|
||||
continue
|
||||
}
|
||||
backend := types.StorageBackendFromLegacy(tenant.ID, provider, legacy)
|
||||
if backend == nil && provider == defaultProvider {
|
||||
backend = types.StorageBackendFromEnvironment(tenant.ID)
|
||||
}
|
||||
if backend == nil {
|
||||
continue
|
||||
}
|
||||
if err := db.Create(backend).Error; err != nil {
|
||||
logger.Warnf(context.Background(), "Failed to migrate %s storage for workspace %d: %v", provider, tenant.ID, err)
|
||||
continue
|
||||
}
|
||||
backendIDs[provider] = backend.ID
|
||||
}
|
||||
if tenant.DefaultStorageBackendID == nil {
|
||||
if id := backendIDs[defaultProvider]; id != "" {
|
||||
if err := db.Model(&types.Tenant{}).Where("id = ?", tenant.ID).Update("default_storage_backend_id", id).Error; err != nil {
|
||||
logger.Warnf(context.Background(), "Failed to set default storage backend for workspace %d: %v", tenant.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var kbs []*types.KnowledgeBase
|
||||
if err := db.Where("tenant_id = ? AND storage_backend_id IS NULL", tenant.ID).Find(&kbs).Error; err != nil {
|
||||
continue
|
||||
}
|
||||
for _, kb := range kbs {
|
||||
provider := kb.GetStorageProvider()
|
||||
if provider == "" {
|
||||
provider = defaultProvider
|
||||
}
|
||||
if id := backendIDs[provider]; id != "" {
|
||||
_ = db.Model(&types.KnowledgeBase{}).Where("id = ? AND storage_backend_id IS NULL", kb.ID).Update("storage_backend_id", id).Error
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// syncSequences ensures PostgreSQL sequences for auto-increment columns (seq_id)
|
||||
// are at least as high as the current MAX value in each table. This is needed
|
||||
// because older code assigned seq_id via application-level MAX()+1, which could
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestMigrateLegacyStorageBackends(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
// Schema is owned by the SQL migrations in production; provision it here so
|
||||
// the test exercises only the data-migration logic.
|
||||
require.NoError(t, db.Exec(`CREATE TABLE tenants (
|
||||
id INTEGER PRIMARY KEY, name TEXT, storage_engine_config TEXT,
|
||||
default_storage_backend_id TEXT, updated_at DATETIME, deleted_at DATETIME
|
||||
)`).Error)
|
||||
require.NoError(t, db.Exec(`CREATE TABLE knowledge_bases (
|
||||
id TEXT PRIMARY KEY, tenant_id INTEGER, storage_provider_config TEXT,
|
||||
storage_backend_id TEXT, cos_config TEXT, updated_at DATETIME, deleted_at DATETIME
|
||||
)`).Error)
|
||||
require.NoError(t, db.AutoMigrate(&types.StorageBackend{}))
|
||||
tenantConfig, err := (&types.StorageEngineConfig{DefaultProvider: "local", Local: &types.LocalEngineConfig{PathPrefix: "workspace-a"}}).Value()
|
||||
require.NoError(t, err)
|
||||
providerConfig, err := (types.StorageProviderConfig{Provider: "local"}).Value()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Exec("INSERT INTO tenants(id, name, storage_engine_config) VALUES (?, ?, ?)", 7, "workspace", tenantConfig).Error)
|
||||
require.NoError(t, db.Exec("INSERT INTO knowledge_bases(id, tenant_id, storage_provider_config, cos_config) VALUES (?, ?, ?, ?)", "kb-a", 7, providerConfig, []byte("{}")).Error)
|
||||
|
||||
migrateLegacyStorageBackends(db)
|
||||
|
||||
var backend types.StorageBackend
|
||||
require.NoError(t, db.Where("tenant_id = ? AND provider = ? AND legacy_alias = ?", 7, "local", true).First(&backend).Error)
|
||||
assert.Equal(t, "workspace-a", backend.Config.PathPrefix)
|
||||
|
||||
var tenantDefault, kbBackend string
|
||||
require.NoError(t, db.Raw("SELECT default_storage_backend_id FROM tenants WHERE id = 7").Scan(&tenantDefault).Error)
|
||||
require.NoError(t, db.Raw("SELECT storage_backend_id FROM knowledge_bases WHERE id = 'kb-a'").Scan(&kbBackend).Error)
|
||||
assert.Equal(t, backend.ID, tenantDefault)
|
||||
assert.Equal(t, backend.ID, kbBackend)
|
||||
}
|
||||
@@ -63,6 +63,7 @@ type InitializationHandler struct {
|
||||
ollamaService *ollama.OllamaService
|
||||
documentReader interfaces.DocumentReader
|
||||
pooler embedding.EmbedderPooler
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
}
|
||||
|
||||
// NewInitializationHandler 创建初始化处理器
|
||||
@@ -76,6 +77,7 @@ func NewInitializationHandler(
|
||||
ollamaService *ollama.OllamaService,
|
||||
documentReader interfaces.DocumentReader,
|
||||
pooler embedding.EmbedderPooler,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
) *InitializationHandler {
|
||||
return &InitializationHandler{
|
||||
config: config,
|
||||
@@ -87,6 +89,7 @@ func NewInitializationHandler(
|
||||
ollamaService: ollamaService,
|
||||
documentReader: documentReader,
|
||||
pooler: pooler,
|
||||
storageResolver: storageResolver,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,7 +126,8 @@ type KBModelConfigRequest struct {
|
||||
} `json:"multimodal"`
|
||||
|
||||
// 存储引擎选择("local" | "minio" | "cos"),影响文档上传与文档内图片存储,参数从全局设置读取
|
||||
StorageProvider string `json:"storageProvider"`
|
||||
StorageProvider string `json:"storageProvider"`
|
||||
StorageBackendID string `json:"storageBackendId"`
|
||||
|
||||
// 知识图谱配置
|
||||
NodeExtract struct {
|
||||
@@ -362,7 +366,30 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
|
||||
kb.VLMConfig.CustomInstructions = strings.TrimSpace(req.VLMConfig.CustomInstructions)
|
||||
}
|
||||
|
||||
// 存储引擎:仅写入 provider 到新字段,参数从空间全局 StorageEngineConfig 读取
|
||||
// Bind the concrete storage instance. Provider remains a compatibility
|
||||
// projection for older clients and historical rows.
|
||||
if strings.TrimSpace(req.StorageBackendID) != "" {
|
||||
tenant, _ := types.TenantInfoFromContext(ctx)
|
||||
backend, resolveErr := h.storageResolver.ResolveBackend(ctx, tenant, req.StorageBackendID, "")
|
||||
if resolveErr != nil || backend == nil {
|
||||
c.Error(errors.NewBadRequestError("Storage backend is unavailable"))
|
||||
return
|
||||
}
|
||||
oldID := ""
|
||||
if kb.StorageBackendID != nil {
|
||||
oldID = *kb.StorageBackendID
|
||||
}
|
||||
if oldID != "" && oldID != backend.ID {
|
||||
knowledgeList, listErr := h.knowledgeService.ListPagedKnowledgeByKnowledgeBaseID(ctx, kbIdStr, &types.Pagination{Page: 1, PageSize: 1}, types.KnowledgeListFilter{})
|
||||
if listErr == nil && knowledgeList != nil && knowledgeList.Total > 0 {
|
||||
c.Error(errors.NewBadRequestError("Storage backend cannot be changed while the knowledge base contains files; migrate storage first"))
|
||||
return
|
||||
}
|
||||
}
|
||||
kb.StorageBackendID = &backend.ID
|
||||
req.StorageProvider = backend.Provider
|
||||
}
|
||||
// Legacy provider projection.
|
||||
provider := strings.ToLower(strings.TrimSpace(req.StorageProvider))
|
||||
if provider == "" {
|
||||
provider = "local"
|
||||
|
||||
@@ -1055,21 +1055,19 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
|
||||
"cross-store cloning is not yet supported"))
|
||||
return
|
||||
}
|
||||
// Pre-flight defense 3: storage backend must match — only meaningful
|
||||
// when the tenant has a StorageEngineConfig. Without it,
|
||||
// resolveFileService ignores per-KB provider pins and routes ALL KBs to
|
||||
// the global storage service, so a clone can never span two real
|
||||
// backends and the pins must NOT be used to reject (false positive).
|
||||
// When a tenant config exists, pins are honored, so reject a genuine
|
||||
// cross-backend clone before enqueueing.
|
||||
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
tenantDefault := tenant.StorageEngineConfig.DefaultProvider
|
||||
srcProvider := sourceKB.EffectiveStorageProvider(tenantDefault)
|
||||
dstProvider := targetKB.EffectiveStorageProvider(tenantDefault)
|
||||
if srcProvider != "" && dstProvider != "" && srcProvider != dstProvider {
|
||||
// Pre-flight defense 3: compare concrete instance IDs, not just the
|
||||
// provider type (COS-A and COS-B are different physical stores).
|
||||
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil {
|
||||
defaultID, defaultProvider := "", ""
|
||||
if tenant.DefaultStorageBackendID != nil {
|
||||
defaultID = *tenant.DefaultStorageBackendID
|
||||
}
|
||||
if tenant.StorageEngineConfig != nil {
|
||||
defaultProvider = tenant.StorageEngineConfig.DefaultProvider
|
||||
}
|
||||
if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) {
|
||||
c.Error(apperrors.NewBadRequestError(
|
||||
"source and target knowledge bases use different storage backends (" +
|
||||
srcProvider + " vs " + dstProvider + "); cross-storage-backend cloning is not supported"))
|
||||
"source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported"))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,9 +27,10 @@ type Handler struct {
|
||||
agentShareService interfaces.AgentShareService // Service for resolving shared agents (KB scope in retrieval)
|
||||
kbShareService interfaces.KBShareService // Service for resolving shared KB permissions
|
||||
fileService interfaces.FileService // Service for file storage (image uploads)
|
||||
modelService interfaces.ModelService // Service for model management (VLM access)
|
||||
userService interfaces.UserService // Service for resolving per-user preferences (e.g. enable_memory default)
|
||||
attachmentProcessor *AttachmentProcessor // Processor for file attachments
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
modelService interfaces.ModelService // Service for model management (VLM access)
|
||||
userService interfaces.UserService // Service for resolving per-user preferences (e.g. enable_memory default)
|
||||
attachmentProcessor *AttachmentProcessor // Processor for file attachments
|
||||
}
|
||||
|
||||
// NewHandler creates a new instance of Handler with all necessary dependencies
|
||||
@@ -45,6 +46,7 @@ func NewHandler(
|
||||
agentShareService interfaces.AgentShareService,
|
||||
kbShareService interfaces.KBShareService,
|
||||
fileService interfaces.FileService,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
modelService interfaces.ModelService,
|
||||
userService interfaces.UserService,
|
||||
documentReader interfaces.DocumentReader,
|
||||
@@ -62,6 +64,7 @@ func NewHandler(
|
||||
agentShareService: agentShareService,
|
||||
kbShareService: kbShareService,
|
||||
fileService: fileService,
|
||||
storageResolver: storageResolver,
|
||||
modelService: modelService,
|
||||
userService: userService,
|
||||
attachmentProcessor: NewAttachmentProcessor(
|
||||
|
||||
@@ -141,12 +141,21 @@ func mimeToExt(mime string) string {
|
||||
}
|
||||
|
||||
func (h *Handler) resolveImageFileService(ctx context.Context, storageProvider string) interfaces.FileService {
|
||||
if strings.TrimSpace(storageProvider) == "" {
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant == nil {
|
||||
return h.fileService
|
||||
}
|
||||
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant == nil || tenant.StorageEngineConfig == nil {
|
||||
if h.storageResolver != nil {
|
||||
svc, resolvedProvider, err := h.storageResolver.ResolveFileService(ctx, tenant, "", storageProvider, "")
|
||||
if err == nil && svc != nil {
|
||||
logger.Infof(ctx, "[image-storage] using storage instance provider=%s for image uploads", resolvedProvider)
|
||||
return svc
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[image-storage] failed to resolve storage instance for provider=%s: %v", storageProvider, err)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(storageProvider) == "" || tenant.StorageEngineConfig == nil {
|
||||
return h.fileService
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/storageallowlist"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type StorageBackendHandler struct {
|
||||
repo interfaces.StorageBackendRepository
|
||||
service interfaces.StorageBackendService
|
||||
}
|
||||
|
||||
func NewStorageBackendHandler(repo interfaces.StorageBackendRepository, service interfaces.StorageBackendService) *StorageBackendHandler {
|
||||
return &StorageBackendHandler{repo: repo, service: service}
|
||||
}
|
||||
|
||||
type storageBackendRequest struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Provider string `json:"provider" binding:"required"`
|
||||
Config types.StorageBackendConfig `json:"config"`
|
||||
Status string `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
func storageTenantID(c *gin.Context) uint64 { return c.GetUint64(types.TenantIDContextKey.String()) }
|
||||
|
||||
func (h *StorageBackendHandler) List(c *gin.Context) {
|
||||
tenantID := storageTenantID(c)
|
||||
backends, err := h.repo.List(c.Request.Context(), tenantID)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
result := make([]types.StorageBackend, 0, len(backends))
|
||||
for _, backend := range backends {
|
||||
result = append(result, types.NewStorageBackendResponse(backend))
|
||||
}
|
||||
tenant, _ := types.TenantInfoFromContext(c.Request.Context())
|
||||
var defaultID *string
|
||||
if tenant != nil {
|
||||
defaultID = tenant.DefaultStorageBackendID
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": result, "default_storage_backend_id": defaultID})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Get(c *gin.Context) {
|
||||
backend, err := h.repo.GetByID(c.Request.Context(), storageTenantID(c), c.Param("id"))
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
if backend == nil {
|
||||
c.Error(apperrors.NewNotFoundError("storage backend not found"))
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": types.NewStorageBackendResponse(backend)})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Create(c *gin.Context) {
|
||||
var req storageBackendRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.Error(apperrors.NewBadRequestError(err.Error()))
|
||||
return
|
||||
}
|
||||
backend := &types.StorageBackend{TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config, Status: req.Status}
|
||||
if err := h.service.Create(c.Request.Context(), backend); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusCreated, gin.H{"success": true, "data": types.NewStorageBackendResponse(backend)})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Update(c *gin.Context) {
|
||||
var req storageBackendRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.Error(apperrors.NewBadRequestError(err.Error()))
|
||||
return
|
||||
}
|
||||
backend := &types.StorageBackend{ID: c.Param("id"), TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config, Status: req.Status}
|
||||
if err := h.service.Update(c.Request.Context(), backend); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
updated, err := h.repo.GetByID(c.Request.Context(), backend.TenantID, backend.ID)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": types.NewStorageBackendResponse(updated)})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Delete(c *gin.Context) {
|
||||
if err := h.service.Delete(c.Request.Context(), storageTenantID(c), c.Param("id")); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) SetDefault(c *gin.Context) {
|
||||
if err := h.service.SetDefault(c.Request.Context(), storageTenantID(c), c.Param("id")); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) TestRaw(c *gin.Context) {
|
||||
var req storageBackendRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.Error(apperrors.NewBadRequestError(err.Error()))
|
||||
return
|
||||
}
|
||||
backend := &types.StorageBackend{TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config}
|
||||
if err := backend.Validate(); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
if err := h.service.Test(c.Request.Context(), backend); err != nil {
|
||||
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) TestByID(c *gin.Context) {
|
||||
backend, err := h.repo.GetByID(c.Request.Context(), storageTenantID(c), c.Param("id"))
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
if backend == nil {
|
||||
c.Error(apperrors.NewNotFoundError("storage backend not found"))
|
||||
return
|
||||
}
|
||||
if err := h.service.Test(c.Request.Context(), backend); err != nil {
|
||||
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Types(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": storageallowlist.AllowedList()})
|
||||
}
|
||||
@@ -98,6 +98,11 @@ func TestFindIncompleteStorageURL(t *testing.T) {
|
||||
"text minio://",
|
||||
5,
|
||||
},
|
||||
{
|
||||
"partial scoped URL at end",
|
||||
"text storage://backend-a/co",
|
||||
5,
|
||||
},
|
||||
{
|
||||
"no storage URL",
|
||||
"just plain text http://example.com",
|
||||
@@ -291,6 +296,11 @@ func TestHoldbackCutoff(t *testing.T) {
|
||||
"prefix minio://wizard-test/10000/exp",
|
||||
7,
|
||||
},
|
||||
{
|
||||
"bare truncated scoped URL without markdown wrapper",
|
||||
"prefix storage://backend-a/cos://bucket/10000/exp",
|
||||
7,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -111,6 +111,18 @@ func TestRewriteStorageURLs_MinIOFallbackViaGlobal(t *testing.T) {
|
||||
assert.NotContains(t, out, "](minio://")
|
||||
}
|
||||
|
||||
func TestRewriteStorageURLs_ScopedPath(t *testing.T) {
|
||||
stub := &stubIMFileService{
|
||||
getFileURL: func(_ context.Context, filePath string) (string, error) {
|
||||
assert.Equal(t, "storage://backend-a/cos://bucket/ap-test/10000/exports/a.png", filePath)
|
||||
return "https://storage.example/a.png", nil
|
||||
},
|
||||
}
|
||||
input := ""
|
||||
output := rewriteStorageURLs(context.Background(), input, newIMFileServiceResolver(&types.Tenant{}, stub))
|
||||
assert.Contains(t, output, "https://storage.example/a.png")
|
||||
}
|
||||
|
||||
func TestCleanIMContent_MinIOFallbackIntegration(t *testing.T) {
|
||||
stub := &stubIMFileService{
|
||||
getFileURL: func(_ context.Context, _ string) (string, error) {
|
||||
|
||||
+41
-19
@@ -77,8 +77,9 @@ func stripImageXMLTags(s string) string {
|
||||
})
|
||||
}
|
||||
|
||||
// storageSchemeRe matches provider:// URLs used by file storage backends.
|
||||
var storageSchemeRe = regexp.MustCompile(`\b(local|minio|s3|cos|tos|oss)://[^\s)\]>"]+`)
|
||||
// storageSchemeRe matches both legacy provider:// URLs and canonical
|
||||
// storage://<backend-id>/provider:// URLs.
|
||||
var storageSchemeRe = regexp.MustCompile(`\b(?:storage://[0-9A-Za-z_-]+/)?(?:local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]+`)
|
||||
|
||||
// rewriteStorageURLs replaces all provider:// URLs in content with HTTP URLs
|
||||
// obtained from fileService.GetFileURL. URLs that are already HTTP or cannot
|
||||
@@ -127,7 +128,7 @@ func rewriteStorageURLs(ctx context.Context, content string, resolver *imFileSer
|
||||
// incompleteURLSuffixRe matches a provider:// URL that reaches the end of the
|
||||
// string — it may continue in the next chunk.
|
||||
var incompleteURLSuffixRe = regexp.MustCompile(
|
||||
`\b(?:local|minio|s3|cos|tos|oss)://[^\s)\]>"]*$`,
|
||||
`\b(?:storage|local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]*$`,
|
||||
)
|
||||
|
||||
// findIncompleteStorageURL returns the byte offset of a potentially truncated
|
||||
@@ -197,18 +198,19 @@ func holdbackCutoff(chunk string) int {
|
||||
}
|
||||
|
||||
// formatIMOutboundAnswer strips thinking/tool blocks and applies IM content cleanup.
|
||||
func formatIMOutboundAnswer(ctx context.Context, raw string, tenant *types.Tenant, defaultFileSvc interfaces.FileService) string {
|
||||
return cleanIMContent(ctx, FormatIMDisplayContent(raw, StreamDisplayFinal), tenant, defaultFileSvc)
|
||||
func formatIMOutboundAnswer(ctx context.Context, raw string, tenant *types.Tenant, defaultFileSvc interfaces.FileService, storageResolvers ...interfaces.StorageBackendResolver) string {
|
||||
return cleanIMContent(ctx, FormatIMDisplayContent(raw, StreamDisplayFinal), tenant, defaultFileSvc, storageResolvers...)
|
||||
}
|
||||
|
||||
// cleanIMContent applies all IM-specific content transformations:
|
||||
// 1. Collapse <image> XML blocks back to plain markdown
|
||||
// 2. Strip <kb/> and <web/> citation tags
|
||||
// 3. Rewrite provider:// URLs to HTTP URLs (scheme-aware per tenant config)
|
||||
func cleanIMContent(ctx context.Context, content string, tenant *types.Tenant, defaultFileSvc interfaces.FileService) string {
|
||||
func cleanIMContent(ctx context.Context, content string, tenant *types.Tenant, defaultFileSvc interfaces.FileService, storageResolvers ...interfaces.StorageBackendResolver) string {
|
||||
content = stripImageXMLTags(content)
|
||||
content = stripIMCitationTags(content)
|
||||
resolver := newIMFileServiceResolver(tenant, defaultFileSvc)
|
||||
resolver := newIMFileServiceResolver(tenant, defaultFileSvc, storageResolvers...)
|
||||
resolver.ctx = ctx
|
||||
content = rewriteStorageURLs(ctx, content, resolver)
|
||||
return content
|
||||
}
|
||||
@@ -225,20 +227,28 @@ func imLocalStorageBaseDir() string {
|
||||
// for the lifetime of one cleanIMContent / outbound message (avoids re-creating SDK clients
|
||||
// for every URL in a long answer).
|
||||
type imFileServiceResolver struct {
|
||||
tenant *types.Tenant
|
||||
defaultSvc interfaces.FileService
|
||||
cache map[string]interfaces.FileService
|
||||
tenant *types.Tenant
|
||||
defaultSvc interfaces.FileService
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
ctx context.Context
|
||||
cache map[string]interfaces.FileService
|
||||
}
|
||||
|
||||
func newIMFileServiceResolver(tenant *types.Tenant, defaultSvc interfaces.FileService) *imFileServiceResolver {
|
||||
return &imFileServiceResolver{
|
||||
func newIMFileServiceResolver(tenant *types.Tenant, defaultSvc interfaces.FileService, storageResolvers ...interfaces.StorageBackendResolver) *imFileServiceResolver {
|
||||
resolver := &imFileServiceResolver{
|
||||
tenant: tenant,
|
||||
defaultSvc: defaultSvc,
|
||||
ctx: context.Background(),
|
||||
cache: make(map[string]interfaces.FileService),
|
||||
}
|
||||
if len(storageResolvers) > 0 {
|
||||
resolver.storageResolver = storageResolvers[0]
|
||||
}
|
||||
return resolver
|
||||
}
|
||||
|
||||
func (r *imFileServiceResolver) resolve(filePath string) interfaces.FileService {
|
||||
backendID, _, _ := types.ParseStorageBackendPath(filePath)
|
||||
provider := types.ParseProviderScheme(filePath)
|
||||
if provider == "" {
|
||||
if r.tenant != nil && r.tenant.StorageEngineConfig != nil {
|
||||
@@ -248,12 +258,21 @@ func (r *imFileServiceResolver) resolve(filePath string) interfaces.FileService
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if svc, ok := r.cache[provider]; ok {
|
||||
cacheKey := backendID + ":" + provider
|
||||
if svc, ok := r.cache[cacheKey]; ok {
|
||||
return svc
|
||||
}
|
||||
if r.storageResolver != nil && r.tenant != nil {
|
||||
svc, _, err := r.storageResolver.ResolveFileService(r.ctx, r.tenant, backendID, provider, imLocalStorageBaseDir())
|
||||
if err == nil {
|
||||
r.cache[cacheKey] = svc
|
||||
return svc
|
||||
}
|
||||
logger.Warnf(r.ctx, "[IM] resolve storage backend failed: backend_id=%s provider=%s err=%v", backendID, provider, err)
|
||||
}
|
||||
svc := buildIMFileServiceForProvider(r.tenant, provider, r.defaultSvc)
|
||||
if svc != nil {
|
||||
r.cache[provider] = svc
|
||||
r.cache[cacheKey] = svc
|
||||
}
|
||||
return svc
|
||||
}
|
||||
@@ -374,7 +393,8 @@ type Service struct {
|
||||
|
||||
// defaultFileSvc is the process-wide storage backend (STORAGE_TYPE / env).
|
||||
// Used when tenant StorageEngineConfig cannot build a service for the URL scheme.
|
||||
defaultFileSvc interfaces.FileService
|
||||
defaultFileSvc interfaces.FileService
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
|
||||
// cmdRegistry holds all registered slash-commands.
|
||||
cmdRegistry *CommandRegistry
|
||||
@@ -787,6 +807,7 @@ func NewService(
|
||||
oauthManager *mcppkg.OAuthManager,
|
||||
redisClient *redis.Client,
|
||||
appCfg *config.Config,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
) *Service {
|
||||
// Resolve IM configuration with defaults.
|
||||
workers, maxQueue, maxPerUser, globalMaxWorkers, rlWindow, rlMax := resolveIMConfig(appCfg)
|
||||
@@ -811,6 +832,7 @@ func NewService(
|
||||
modelService: modelService,
|
||||
streamManager: streamManager,
|
||||
defaultFileSvc: defaultFileSvc,
|
||||
storageResolver: storageResolver,
|
||||
oauthManager: oauthManager,
|
||||
cmdRegistry: registry,
|
||||
channels: make(map[string]*channelState),
|
||||
@@ -1602,7 +1624,7 @@ func (s *Service) executeQARequest(req *qaRequest) {
|
||||
}
|
||||
|
||||
reply := &ReplyMessage{
|
||||
Content: formatIMOutboundAnswer(ctx, answer, req.tenant, s.defaultFileSvc),
|
||||
Content: formatIMOutboundAnswer(ctx, answer, req.tenant, s.defaultFileSvc, s.storageResolver),
|
||||
IsFinal: true,
|
||||
}
|
||||
if err := req.adapter.SendReply(ctx, req.msg, reply); err != nil {
|
||||
@@ -2357,7 +2379,7 @@ func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage,
|
||||
displaySource = displaySource[:cut]
|
||||
}
|
||||
|
||||
display := cleanIMContent(ctx, displaySource, tenant, s.defaultFileSvc)
|
||||
display := cleanIMContent(ctx, displaySource, tenant, s.defaultFileSvc, s.storageResolver)
|
||||
if err := streamer.UpdateStreamContent(ctx, msg, streamID, display); err != nil {
|
||||
logger.Warnf(ctx, "[IM] UpdateStreamContent failed: %v", err)
|
||||
}
|
||||
@@ -2396,7 +2418,7 @@ loop:
|
||||
authServices := append([]imMCPAuthService(nil), mcpAuthServices...)
|
||||
bufMu.Unlock()
|
||||
|
||||
finalDisplay := cleanIMContent(ctx, FormatIMFinalFromParts(parts), tenant, s.defaultFileSvc)
|
||||
finalDisplay := cleanIMContent(ctx, FormatIMFinalFromParts(parts), tenant, s.defaultFileSvc, s.storageResolver)
|
||||
if noVisibleContent || finalDisplay == "" {
|
||||
fallback := "抱歉,我暂时无法回答这个问题。"
|
||||
if finalErr != nil {
|
||||
@@ -2443,7 +2465,7 @@ func (s *Service) fallbackNonStream(ctx context.Context, msg *IncomingMessage, s
|
||||
answer = "抱歉,处理您的问题时出现了异常,请稍后再试。"
|
||||
}
|
||||
|
||||
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: formatIMOutboundAnswer(ctx, answer, tenant, s.defaultFileSvc), IsFinal: true})
|
||||
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: formatIMOutboundAnswer(ctx, answer, tenant, s.defaultFileSvc, s.storageResolver), IsFinal: true})
|
||||
}
|
||||
|
||||
// runQA executes the WeKnora QA pipeline and returns the full answer text.
|
||||
|
||||
@@ -297,6 +297,13 @@ func apiKeyManageVectorStores(base middleware.APIKeyRoutePolicy) middleware.APIK
|
||||
return base.WithCapability(types.APIKeyCapabilityManageVectorStores)
|
||||
}
|
||||
|
||||
// apiKeyManageStorageBackends layers the "manage_storage_backends" capability
|
||||
// on top of a base policy so a scoped key can manage object/file storage
|
||||
// backend instances without carrying vector-store or full tenant access.
|
||||
func apiKeyManageStorageBackends(base middleware.APIKeyRoutePolicy) middleware.APIKeyRoutePolicy {
|
||||
return base.WithCapability(types.APIKeyCapabilityManageStorageBackends)
|
||||
}
|
||||
|
||||
func apiKeyManageWebSearch(base middleware.APIKeyRoutePolicy) middleware.APIKeyRoutePolicy {
|
||||
return base.WithCapability(types.APIKeyCapabilityManageWebSearch)
|
||||
}
|
||||
|
||||
+93
-22
@@ -74,6 +74,8 @@ type RouterParams struct {
|
||||
WebSearchProviderHandler *handler.WebSearchProviderHandler
|
||||
WebSearchCredentialsHandler *handler.WebSearchProviderCredentialsHandler
|
||||
VectorStoreHandler *handler.VectorStoreHandler
|
||||
StorageBackendHandler *handler.StorageBackendHandler
|
||||
StorageBackendResolver interfaces.StorageBackendResolver
|
||||
FAQHandler *handler.FAQHandler
|
||||
TagHandler *handler.TagHandler
|
||||
CustomAgentHandler *handler.CustomAgentHandler
|
||||
@@ -156,19 +158,19 @@ func NewRouter(params RouterParams) *gin.Engine {
|
||||
RegisterIMRoutes(r, params.IMHandler)
|
||||
|
||||
// Web embed 公开路由(使用 publish token 鉴权,不走全局 Auth)
|
||||
RegisterEmbedPublicRoutes(r, params.EmbedChannelHandler, params.EmbedChannelService, params.TenantService, params.RedisClient, params.FileService)
|
||||
RegisterEmbedPublicRoutes(r, params.EmbedChannelHandler, params.EmbedChannelService, params.TenantService, params.RedisClient, params.FileService, params.StorageBackendResolver)
|
||||
|
||||
// 认证中间件
|
||||
r.Use(middleware.Auth(params.TenantService, params.UserService, params.TenantMemberService, params.TenantAPIKeyService, params.Config))
|
||||
|
||||
// 文件服务:统一代理本地/MinIO/COS/TOS存储后端(需要认证)
|
||||
serveFiles(r, params.FileService)
|
||||
serveFiles(r, params.FileService, params.StorageBackendResolver)
|
||||
|
||||
// Presigned file access: no auth required, signature-verified.
|
||||
servePresignedFiles(r, params.TenantService)
|
||||
servePresignedFiles(r, params.TenantService, params.StorageBackendResolver)
|
||||
|
||||
// Diagnostic preview of presigned URLs (Admin only, behind auth middleware).
|
||||
servePresignedPreview(r, params.Config)
|
||||
servePresignedPreview(r, params.Config, params.StorageBackendResolver)
|
||||
|
||||
// Langfuse observability — only active when LANGFUSE_* env vars are set.
|
||||
// The middleware is registered unconditionally; when disabled it's a no-op.
|
||||
@@ -217,7 +219,7 @@ func NewRouter(params RouterParams) *gin.Engine {
|
||||
// KB-scoped image proxy: lets tenants render images embedded in
|
||||
// org-shared / agent-visible KB content, which the tenant-scoped
|
||||
// /files route cannot serve because it enforces same-tenant paths.
|
||||
serveKBScopedFiles(v1, rbacGuards, params.TenantService, params.FileService)
|
||||
serveKBScopedFiles(v1, rbacGuards, params.TenantService, params.FileService, params.StorageBackendResolver)
|
||||
RegisterKnowledgeTagRoutes(v1, params.TagHandler, rbacGuards)
|
||||
RegisterKnowledgeRoutes(v1, params.KnowledgeHandler, rbacGuards)
|
||||
RegisterFAQRoutes(v1, params.FAQHandler, rbacGuards)
|
||||
@@ -234,6 +236,7 @@ func NewRouter(params RouterParams) *gin.Engine {
|
||||
RegisterWebSearchRoutes(v1, params.WebSearchHandler, rbacGuards)
|
||||
RegisterWebSearchProviderRoutes(v1, params.WebSearchProviderHandler, params.WebSearchCredentialsHandler, rbacGuards)
|
||||
RegisterVectorStoreRoutes(v1, params.VectorStoreHandler, rbacGuards)
|
||||
RegisterStorageBackendRoutes(v1, params.StorageBackendHandler, rbacGuards)
|
||||
RegisterCustomAgentRoutes(v1, params.CustomAgentHandler, rbacGuards)
|
||||
RegisterUserFavoriteRoutes(v1, params.UserFavoriteHandler, rbacGuards)
|
||||
RegisterSkillRoutes(v1, params.SkillHandler, rbacGuards)
|
||||
@@ -1075,6 +1078,22 @@ func RegisterVectorStoreRoutes(r *gin.RouterGroup, h *handler.VectorStoreHandler
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterStorageBackendRoutes manages concrete object/file storage instances.
|
||||
func RegisterStorageBackendRoutes(r *gin.RouterGroup, h *handler.StorageBackendHandler, g *rbacGuards) {
|
||||
backends := g.apiKeyGroup(r.Group("/storage-backends"), apiKeyManageStorageBackends(apiKeyFullAccess()))
|
||||
{
|
||||
backends.GET("/types", g.Viewer(), h.Types)
|
||||
backends.POST("/test", g.Admin(), h.TestRaw)
|
||||
backends.POST("", g.Admin(), h.Create)
|
||||
backends.GET("", g.Viewer(), h.List)
|
||||
backends.GET("/:id", g.Viewer(), h.Get)
|
||||
backends.PUT("/:id", g.Admin(), h.Update)
|
||||
backends.DELETE("/:id", g.Admin(), h.Delete)
|
||||
backends.POST("/:id/test", g.Admin(), h.TestByID)
|
||||
backends.PUT("/:id/default", g.Admin(), h.SetDefault)
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterCustomAgentRoutes registers custom agent routes.
|
||||
//
|
||||
// Mutating routes use OwnedAgentOrAdmin: the original creator can edit
|
||||
@@ -1275,6 +1294,7 @@ func RegisterEmbedPublicRoutes(
|
||||
tenantService interfaces.TenantService,
|
||||
redisClient *redis.Client,
|
||||
fileService interfaces.FileService,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
) {
|
||||
if embedHandler == nil || embedService == nil {
|
||||
return
|
||||
@@ -1302,7 +1322,7 @@ func RegisterEmbedPublicRoutes(
|
||||
// Serve images embedded in bot replies (e.g. chart exports). EmbedAuth
|
||||
// injects the channel's tenant, and the handler enforces that the
|
||||
// requested path belongs to that tenant.
|
||||
embed.GET("/files", newFileServeHandler(fileService))
|
||||
embed.GET("/files", newFileServeHandler(fileService, storageResolver))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1522,7 +1542,7 @@ type getRouteRegistrar interface {
|
||||
// same handler backs both the authenticated /files route and the embed route
|
||||
// (where EmbedAuth injects the channel's tenant). Tenant ownership of the
|
||||
// requested path is enforced via ValidateStoragePathTenant either way.
|
||||
func newFileServeHandler(globalFileService interfaces.FileService) gin.HandlerFunc {
|
||||
func newFileServeHandler(globalFileService interfaces.FileService, storageResolver interfaces.StorageBackendResolver) gin.HandlerFunc {
|
||||
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
|
||||
if baseDir == "" {
|
||||
baseDir = "/data/files"
|
||||
@@ -1545,7 +1565,12 @@ func newFileServeHandler(globalFileService interfaces.FileService) gin.HandlerFu
|
||||
return
|
||||
}
|
||||
|
||||
provider := types.ParseProviderScheme(filePath)
|
||||
backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath)
|
||||
providerPath := filePath
|
||||
if scoped {
|
||||
providerPath = innerPath
|
||||
}
|
||||
provider := types.ParseProviderScheme(providerPath)
|
||||
|
||||
tenant, _ := c.Request.Context().Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant == nil {
|
||||
@@ -1565,7 +1590,9 @@ func newFileServeHandler(globalFileService interfaces.FileService) gin.HandlerFu
|
||||
err error
|
||||
)
|
||||
|
||||
if tenant.StorageEngineConfig != nil {
|
||||
if storageResolver != nil {
|
||||
fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(c.Request.Context(), tenant, backendID, provider, absDir)
|
||||
} else if tenant.StorageEngineConfig != nil {
|
||||
fileSvc, resolvedProvider, err = filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
|
||||
} else {
|
||||
err = http.ErrMissingFile
|
||||
@@ -1608,12 +1635,16 @@ func newFileServeHandler(globalFileService interfaces.FileService) gin.HandlerFu
|
||||
}
|
||||
}
|
||||
|
||||
func serveFiles(r getRouteRegistrar, globalFileService interfaces.FileService) {
|
||||
func serveFiles(r getRouteRegistrar, globalFileService interfaces.FileService, resolvers ...interfaces.StorageBackendResolver) {
|
||||
logger.Infof(context.Background(), "[Router] Serving files from /files")
|
||||
var storageResolver interfaces.StorageBackendResolver
|
||||
if len(resolvers) > 0 {
|
||||
storageResolver = resolvers[0]
|
||||
}
|
||||
// /files sits outside the /api/v1 APIKeyGate; storage paths cannot be
|
||||
// attributed to a key's KB allow-list, so API keys are denied outright
|
||||
// (embed routes use their own /embed/.../files handler).
|
||||
r.GET("/files", middleware.DenyAPIKeyPrincipal(), newFileServeHandler(globalFileService))
|
||||
r.GET("/files", middleware.DenyAPIKeyPrincipal(), newFileServeHandler(globalFileService, storageResolver))
|
||||
}
|
||||
|
||||
// serveKBScopedFiles registers the KB-scoped file proxy used to render images
|
||||
@@ -1633,6 +1664,7 @@ func serveKBScopedFiles(
|
||||
g *rbacGuards,
|
||||
tenantService interfaces.TenantService,
|
||||
globalFileService interfaces.FileService,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
) {
|
||||
logger.Infof(context.Background(), "[Router] Serving KB-scoped files from /knowledge-bases/:id/files")
|
||||
// API keys are denied outright (as on /files): a signed URL's tenant/KB
|
||||
@@ -1642,7 +1674,7 @@ func serveKBScopedFiles(
|
||||
middleware.DenyAPIKeyPrincipal(),
|
||||
g.Viewer(),
|
||||
g.KBAccessRead("id"),
|
||||
newKBScopedFileServeHandler(tenantService, globalFileService),
|
||||
newKBScopedFileServeHandler(tenantService, globalFileService, storageResolver),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1655,7 +1687,12 @@ func serveKBScopedFiles(
|
||||
func newKBScopedFileServeHandler(
|
||||
tenantService interfaces.TenantService,
|
||||
globalFileService interfaces.FileService,
|
||||
resolvers ...interfaces.StorageBackendResolver,
|
||||
) gin.HandlerFunc {
|
||||
var storageResolver interfaces.StorageBackendResolver
|
||||
if len(resolvers) > 0 {
|
||||
storageResolver = resolvers[0]
|
||||
}
|
||||
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
|
||||
if baseDir == "" {
|
||||
baseDir = "/data/files"
|
||||
@@ -1700,13 +1737,20 @@ func newKBScopedFileServeHandler(
|
||||
return
|
||||
}
|
||||
|
||||
provider := types.ParseProviderScheme(filePath)
|
||||
backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath)
|
||||
providerPath := filePath
|
||||
if scoped {
|
||||
providerPath = innerPath
|
||||
}
|
||||
provider := types.ParseProviderScheme(providerPath)
|
||||
|
||||
var (
|
||||
fileSvc interfaces.FileService
|
||||
resolvedProvider string
|
||||
)
|
||||
if tenant.StorageEngineConfig != nil {
|
||||
if storageResolver != nil {
|
||||
fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(ctx, tenant, backendID, provider, absDir)
|
||||
} else if tenant.StorageEngineConfig != nil {
|
||||
fileSvc, resolvedProvider, err = filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
|
||||
} else {
|
||||
err = http.ErrMissingFile
|
||||
@@ -1766,14 +1810,14 @@ func newKBScopedFileServeHandler(
|
||||
// Without this it is otherwise impossible to tell whether a "broken image" is
|
||||
// caused by an expired signature, a stale URL cached by the platform, the
|
||||
// platform's IP being blocked, or the URL simply never reaching us.
|
||||
func servePresignedFiles(r *gin.Engine, tenantService interfaces.TenantService) {
|
||||
func servePresignedFiles(r *gin.Engine, tenantService interfaces.TenantService, storageResolver interfaces.StorageBackendResolver) {
|
||||
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
|
||||
if baseDir == "" {
|
||||
baseDir = "/data/files"
|
||||
}
|
||||
absDir, _ := filepath.Abs(baseDir)
|
||||
|
||||
handler := presignedFileHandler(tenantService, absDir)
|
||||
handler := presignedFileHandler(tenantService, absDir, storageResolver)
|
||||
r.GET("/api/v1/files/presigned", handler)
|
||||
r.HEAD("/api/v1/files/presigned", handler)
|
||||
}
|
||||
@@ -1782,7 +1826,11 @@ func servePresignedFiles(r *gin.Engine, tenantService interfaces.TenantService)
|
||||
// For HEAD requests it returns the same status + headers but does not stream
|
||||
// the body — this is enough for IM platforms to validate the URL while saving
|
||||
// us a full read of the backing object.
|
||||
func presignedFileHandler(tenantService interfaces.TenantService, absDir string) gin.HandlerFunc {
|
||||
func presignedFileHandler(tenantService interfaces.TenantService, absDir string, resolvers ...interfaces.StorageBackendResolver) gin.HandlerFunc {
|
||||
var storageResolver interfaces.StorageBackendResolver
|
||||
if len(resolvers) > 0 {
|
||||
storageResolver = resolvers[0]
|
||||
}
|
||||
return func(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
clientIP := c.ClientIP()
|
||||
@@ -1823,7 +1871,12 @@ func presignedFileHandler(tenantService interfaces.TenantService, absDir string)
|
||||
return
|
||||
}
|
||||
|
||||
provider := types.ParseProviderScheme(filePath)
|
||||
backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath)
|
||||
providerPath := filePath
|
||||
if scoped {
|
||||
providerPath = innerPath
|
||||
}
|
||||
provider := types.ParseProviderScheme(providerPath)
|
||||
tenant, err := tenantService.GetTenantByID(ctx, tenantID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Router] /files/presigned tenant lookup failed: client_ip=%s tenant_id=%d err=%v", clientIP, tenantID, err)
|
||||
@@ -1831,7 +1884,13 @@ func presignedFileHandler(tenantService interfaces.TenantService, absDir string)
|
||||
return
|
||||
}
|
||||
|
||||
fileSvc, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
|
||||
var fileSvc interfaces.FileService
|
||||
var resolvedProvider string
|
||||
if storageResolver != nil {
|
||||
fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(ctx, tenant, backendID, provider, absDir)
|
||||
} else {
|
||||
fileSvc, resolvedProvider, err = filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Router] /files/presigned resolve file service failed: client_ip=%s tenant_id=%d provider=%s err=%v",
|
||||
clientIP, tenantID, provider, err)
|
||||
@@ -1881,7 +1940,7 @@ func presignedFileHandler(tenantService interfaces.TenantService, absDir string)
|
||||
//
|
||||
// Route:
|
||||
// - GET /api/v1/files/presigned-preview?file_path=<provider://...>
|
||||
func servePresignedPreview(r *gin.Engine, cfg *config.Config) {
|
||||
func servePresignedPreview(r *gin.Engine, cfg *config.Config, storageResolver interfaces.StorageBackendResolver) {
|
||||
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
|
||||
if baseDir == "" {
|
||||
baseDir = "/data/files"
|
||||
@@ -1913,8 +1972,20 @@ func servePresignedPreview(r *gin.Engine, cfg *config.Config) {
|
||||
return
|
||||
}
|
||||
|
||||
provider := types.ParseProviderScheme(filePath)
|
||||
fileSvc, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
|
||||
backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath)
|
||||
providerPath := filePath
|
||||
if scoped {
|
||||
providerPath = innerPath
|
||||
}
|
||||
provider := types.ParseProviderScheme(providerPath)
|
||||
var fileSvc interfaces.FileService
|
||||
var resolvedProvider string
|
||||
var err error
|
||||
if storageResolver != nil {
|
||||
fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(ctx, tenant, backendID, provider, absDir)
|
||||
} else {
|
||||
fileSvc, resolvedProvider, err = filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir)
|
||||
}
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"error": err.Error(),
|
||||
|
||||
@@ -273,6 +273,7 @@ func TestTenantInfrastructureRoutesDeclareSpecificCapabilities(t *testing.T) {
|
||||
RegisterMCPServiceRoutes(v1, &handler.MCPServiceHandler{}, &handler.MCPCredentialsHandler{}, &handler.MCPOAuthHandler{}, g)
|
||||
RegisterWebSearchProviderRoutes(v1, &handler.WebSearchProviderHandler{}, &handler.WebSearchProviderCredentialsHandler{}, g)
|
||||
RegisterVectorStoreRoutes(v1, &handler.VectorStoreHandler{}, g)
|
||||
RegisterStorageBackendRoutes(v1, &handler.StorageBackendHandler{}, g)
|
||||
RegisterEmbedChannelRoutes(v1, &handler.EmbedChannelHandler{}, g)
|
||||
RegisterIMChannelRoutes(v1, &handler.IMHandler{}, g)
|
||||
RegisterDataSourceRoutes(v1, &handler.DataSourceHandler{}, &handler.DataSourceCredentialsHandler{}, g)
|
||||
@@ -290,6 +291,7 @@ func TestTenantInfrastructureRoutesDeclareSpecificCapabilities(t *testing.T) {
|
||||
{http.MethodGet, "/api/v1/mcp-services", types.APIKeyCapabilityManageMCPServices},
|
||||
{http.MethodGet, "/api/v1/web-search-providers", types.APIKeyCapabilityManageWebSearch},
|
||||
{http.MethodGet, "/api/v1/vector-stores", types.APIKeyCapabilityManageVectorStores},
|
||||
{http.MethodGet, "/api/v1/storage-backends", types.APIKeyCapabilityManageStorageBackends},
|
||||
{http.MethodGet, "/api/v1/embed-channels", types.APIKeyCapabilityManageChannels},
|
||||
{http.MethodGet, "/api/v1/im-channels", types.APIKeyCapabilityManageChannels},
|
||||
{http.MethodGet, "/api/v1/datasource", types.APIKeyCapabilityManageDataSources},
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package interfaces
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
type StorageBackendRepository interface {
|
||||
Create(ctx context.Context, backend *types.StorageBackend) error
|
||||
GetByID(ctx context.Context, tenantID uint64, id string) (*types.StorageBackend, error)
|
||||
List(ctx context.Context, tenantID uint64) ([]*types.StorageBackend, error)
|
||||
Update(ctx context.Context, backend *types.StorageBackend) error
|
||||
Delete(ctx context.Context, tenantID uint64, id string) error
|
||||
FindLegacyAlias(ctx context.Context, tenantID uint64, provider string) (*types.StorageBackend, error)
|
||||
}
|
||||
|
||||
type StorageBackendService interface {
|
||||
Create(ctx context.Context, backend *types.StorageBackend) error
|
||||
Update(ctx context.Context, backend *types.StorageBackend) error
|
||||
Delete(ctx context.Context, tenantID uint64, id string) error
|
||||
SetDefault(ctx context.Context, tenantID uint64, id string) error
|
||||
Test(ctx context.Context, backend *types.StorageBackend) error
|
||||
}
|
||||
|
||||
// StorageBackendResolver is the single runtime entry point for resolving one
|
||||
// concrete storage instance. backendID wins; provider is a legacy fallback.
|
||||
type StorageBackendResolver interface {
|
||||
ResolveFileService(ctx context.Context, tenant *types.Tenant, backendID, provider, localBaseDir string) (FileService, string, error)
|
||||
ResolveBackend(ctx context.Context, tenant *types.Tenant, backendID, provider string) (*types.StorageBackend, error)
|
||||
}
|
||||
@@ -9,6 +9,24 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const storageBackendScheme = "storage://"
|
||||
|
||||
func BuildStorageBackendPath(backendID, providerPath string) string {
|
||||
return storageBackendScheme + strings.TrimSpace(backendID) + "/" + providerPath
|
||||
}
|
||||
|
||||
func ParseStorageBackendPath(path string) (backendID, providerPath string, ok bool) {
|
||||
if !strings.HasPrefix(path, storageBackendScheme) {
|
||||
return "", "", false
|
||||
}
|
||||
rest := strings.TrimPrefix(path, storageBackendScheme)
|
||||
parts := strings.SplitN(rest, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
|
||||
// KnowledgeBaseType represents the type of the knowledge base
|
||||
const (
|
||||
// KnowledgeBaseTypeDocument represents the document knowledge base type
|
||||
@@ -71,6 +89,9 @@ type KnowledgeBase struct {
|
||||
ASRConfig ASRConfig `yaml:"asr_config" json:"asr_config" gorm:"type:json"`
|
||||
// Storage provider config (new): only stores provider selection; credentials from workspace StorageEngineConfig
|
||||
StorageProviderConfig *StorageProviderConfig `yaml:"storage_provider_config" json:"storage_provider_config" gorm:"column:storage_provider_config;type:jsonb"`
|
||||
// StorageBackendID binds this KB to one concrete storage instance. The
|
||||
// legacy provider field remains readable during migration only.
|
||||
StorageBackendID *string `yaml:"storage_backend_id" json:"storage_backend_id,omitempty" gorm:"column:storage_backend_id;type:varchar(36);default:null"`
|
||||
// Deprecated: legacy COS config column. Kept for backward compatibility with old data.
|
||||
StorageConfig StorageConfig `yaml:"-" json:"storage_config" gorm:"column:cos_config;type:json"`
|
||||
// VectorStoreID references the VectorStore this knowledge base is bound to.
|
||||
@@ -317,6 +338,27 @@ func (kb *KnowledgeBase) SetStorageProvider(provider string) {
|
||||
kb.StorageProviderConfig = &StorageProviderConfig{Provider: provider}
|
||||
}
|
||||
|
||||
// SharesStorageBackendWith compares concrete instance bindings first. Provider
|
||||
// comparison is only a compatibility fallback for rows not yet backfilled.
|
||||
func (kb *KnowledgeBase) SharesStorageBackendWith(other *KnowledgeBase, defaultBackendID, defaultProvider string) bool {
|
||||
if kb == nil || other == nil {
|
||||
return false
|
||||
}
|
||||
effectiveID := func(candidate *KnowledgeBase) string {
|
||||
if candidate.StorageBackendID != nil {
|
||||
if id := strings.TrimSpace(*candidate.StorageBackendID); id != "" {
|
||||
return id
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(defaultBackendID)
|
||||
}
|
||||
leftID, rightID := effectiveID(kb), effectiveID(other)
|
||||
if leftID != "" || rightID != "" {
|
||||
return leftID != "" && leftID == rightID
|
||||
}
|
||||
return kb.EffectiveStorageProvider(defaultProvider) == other.EffectiveStorageProvider(defaultProvider)
|
||||
}
|
||||
|
||||
// InferStorageFromFilePath deduces the storage provider from a file path format.
|
||||
// Used as a safety fallback when the KB's configured provider doesn't match the data.
|
||||
// Supports provider:// scheme (local://, minio://, cos://, tos://),
|
||||
@@ -339,6 +381,9 @@ func InferStorageFromFilePath(filePath string) string {
|
||||
// e.g. "minio://bucket/key" → "minio", "local://tenant/file.pdf" → "local"
|
||||
// Returns "" if the path does not use a known provider scheme.
|
||||
func ParseProviderScheme(filePath string) string {
|
||||
if _, inner, ok := ParseStorageBackendPath(filePath); ok {
|
||||
filePath = inner
|
||||
}
|
||||
for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3", "obs"} {
|
||||
if strings.HasPrefix(filePath, provider+"://") {
|
||||
return provider
|
||||
|
||||
@@ -0,0 +1,357 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/storageallowlist"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
StorageBackendSourceUser = "user"
|
||||
StorageBackendSourceEnv = "env"
|
||||
StorageBackendStatusActive = "active"
|
||||
StorageBackendStatusDisabled = "disabled"
|
||||
)
|
||||
|
||||
// StorageBackend is one concrete file/object storage instance. A workspace may
|
||||
// register multiple instances of the same provider and bind each knowledge base
|
||||
// to a different instance.
|
||||
type StorageBackend struct {
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
TenantID uint64 `json:"tenant_id" gorm:"not null;index"`
|
||||
Name string `json:"name" gorm:"type:varchar(255);not null"`
|
||||
Provider string `json:"provider" gorm:"type:varchar(32);not null;index"`
|
||||
Config StorageBackendConfig `json:"config" gorm:"type:json"`
|
||||
Source string `json:"source" gorm:"type:varchar(16);not null;default:'user'"`
|
||||
Status string `json:"status" gorm:"type:varchar(16);not null;default:'active'"`
|
||||
LegacyAlias bool `json:"legacy_alias" gorm:"not null;default:false"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
}
|
||||
|
||||
func (StorageBackend) TableName() string { return "storage_backends" }
|
||||
|
||||
func (b *StorageBackend) BeforeCreate(_ *gorm.DB) error {
|
||||
if b.ID == "" {
|
||||
b.ID = uuid.NewString()
|
||||
}
|
||||
if b.Source == "" {
|
||||
b.Source = StorageBackendSourceUser
|
||||
}
|
||||
if b.Status == "" {
|
||||
b.Status = StorageBackendStatusActive
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *StorageBackend) Validate() error {
|
||||
if b.TenantID == 0 {
|
||||
return errors.NewValidationError("tenant_id is required")
|
||||
}
|
||||
b.Name = strings.TrimSpace(b.Name)
|
||||
if b.Name == "" {
|
||||
return errors.NewValidationError("name is required")
|
||||
}
|
||||
b.Provider = strings.ToLower(strings.TrimSpace(b.Provider))
|
||||
if !storageallowlist.IsAllowed(b.Provider) {
|
||||
return errors.NewValidationError(fmt.Sprintf("storage provider %q is not allowed", b.Provider))
|
||||
}
|
||||
if !isSupportedStorageBackendProvider(b.Provider) {
|
||||
return errors.NewValidationError(fmt.Sprintf("unsupported storage provider: %s", b.Provider))
|
||||
}
|
||||
if b.Status == "" {
|
||||
b.Status = StorageBackendStatusActive
|
||||
}
|
||||
if b.Status != StorageBackendStatusActive && b.Status != StorageBackendStatusDisabled {
|
||||
return errors.NewValidationError("status must be active or disabled")
|
||||
}
|
||||
return b.Config.ValidateForProvider(b.Provider)
|
||||
}
|
||||
|
||||
func isSupportedStorageBackendProvider(provider string) bool {
|
||||
for _, candidate := range storageallowlist.Supported() {
|
||||
if provider == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// StorageBackendConfig is the normalized union of provider-specific storage
|
||||
// settings. AccessKeyID/SecretAccessKey map to COS SecretID/SecretKey and to
|
||||
// the access/secret key pair used by S3-compatible providers.
|
||||
type StorageBackendConfig struct {
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Region string `json:"region,omitempty"`
|
||||
AccessKeyID string `json:"access_key_id,omitempty"`
|
||||
SecretAccessKey string `json:"secret_access_key,omitempty"`
|
||||
BucketName string `json:"bucket_name,omitempty"`
|
||||
PathPrefix string `json:"path_prefix,omitempty"`
|
||||
AppID string `json:"app_id,omitempty"`
|
||||
UseSSL bool `json:"use_ssl,omitempty"`
|
||||
ForcePathStyle bool `json:"force_path_style,omitempty"`
|
||||
UseTempBucket bool `json:"use_temp_bucket,omitempty"`
|
||||
TempBucketName string `json:"temp_bucket_name,omitempty"`
|
||||
TempRegion string `json:"temp_region,omitempty"`
|
||||
}
|
||||
|
||||
func (c StorageBackendConfig) Value() (driver.Value, error) {
|
||||
if key := utils.GetAESKey(); key != nil {
|
||||
if c.AccessKeyID != "" {
|
||||
if encrypted, err := utils.EncryptAESGCM(c.AccessKeyID, key); err == nil {
|
||||
c.AccessKeyID = encrypted
|
||||
}
|
||||
}
|
||||
if c.SecretAccessKey != "" {
|
||||
if encrypted, err := utils.EncryptAESGCM(c.SecretAccessKey, key); err == nil {
|
||||
c.SecretAccessKey = encrypted
|
||||
}
|
||||
}
|
||||
}
|
||||
return json.Marshal(c)
|
||||
}
|
||||
|
||||
func (c *StorageBackendConfig) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
b, ok := value.([]byte)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(b, c); err != nil {
|
||||
return err
|
||||
}
|
||||
accessKey, err := utils.DecryptStoredSecret(c.AccessKeyID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt storage backend access key: %w", err)
|
||||
}
|
||||
secretKey, err := utils.DecryptStoredSecret(c.SecretAccessKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt storage backend secret key: %w", err)
|
||||
}
|
||||
c.AccessKeyID = accessKey
|
||||
c.SecretAccessKey = secretKey
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c StorageBackendConfig) MaskSensitiveFields() StorageBackendConfig {
|
||||
out := c
|
||||
if out.AccessKeyID != "" {
|
||||
out.AccessKeyID = RedactedSecretPlaceholder
|
||||
}
|
||||
if out.SecretAccessKey != "" {
|
||||
out.SecretAccessKey = RedactedSecretPlaceholder
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c StorageBackendConfig) MergeSecrets(existing StorageBackendConfig) StorageBackendConfig {
|
||||
c.AccessKeyID = PreserveIfRedacted(c.AccessKeyID, existing.AccessKeyID)
|
||||
c.SecretAccessKey = PreserveIfRedacted(c.SecretAccessKey, existing.SecretAccessKey)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c StorageBackendConfig) ValidateForProvider(provider string) error {
|
||||
prefix := strings.ReplaceAll(strings.TrimSpace(c.PathPrefix), "\\", "/")
|
||||
cleanPrefix := path.Clean(prefix)
|
||||
if strings.HasPrefix(prefix, "/") || cleanPrefix == ".." || strings.HasPrefix(cleanPrefix, "../") {
|
||||
return errors.NewValidationError("path_prefix must be a relative path without parent traversal")
|
||||
}
|
||||
required := func(name, value string) error {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return errors.NewValidationError(name + " is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
switch provider {
|
||||
case "local":
|
||||
return nil
|
||||
case "minio":
|
||||
if c.Mode == "" {
|
||||
c.Mode = "remote"
|
||||
}
|
||||
if c.Mode != "docker" {
|
||||
for name, value := range map[string]string{"endpoint": c.Endpoint, "access_key_id": c.AccessKeyID, "secret_access_key": c.SecretAccessKey} {
|
||||
if err := required(name, value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return required("bucket_name", c.BucketName)
|
||||
case "cos":
|
||||
for name, value := range map[string]string{"region": c.Region, "access_key_id": c.AccessKeyID, "secret_access_key": c.SecretAccessKey, "bucket_name": c.BucketName} {
|
||||
if err := required(name, value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
for name, value := range map[string]string{"endpoint": c.Endpoint, "region": c.Region, "access_key_id": c.AccessKeyID, "secret_access_key": c.SecretAccessKey, "bucket_name": c.BucketName} {
|
||||
if err := required(name, value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// LocationKey identifies the physical destination. Credentials deliberately do
|
||||
// not participate so they can be rotated without changing object identity.
|
||||
func (c StorageBackendConfig) LocationKey(provider string) string {
|
||||
mode := strings.TrimSpace(c.Mode)
|
||||
if provider == "minio" && mode == "" {
|
||||
mode = "remote"
|
||||
}
|
||||
return strings.Join([]string{provider, mode, strings.TrimSpace(c.Endpoint), strings.TrimSpace(c.Region), strings.TrimSpace(c.BucketName), strings.Trim(strings.TrimSpace(c.PathPrefix), "/")}, "|")
|
||||
}
|
||||
|
||||
// ToStorageEngineConfig adapts the instance model to the existing provider
|
||||
// implementations while those implementations are progressively normalized.
|
||||
func (b StorageBackend) ToStorageEngineConfig() *StorageEngineConfig {
|
||||
c := b.Config
|
||||
result := &StorageEngineConfig{DefaultProvider: b.Provider}
|
||||
switch b.Provider {
|
||||
case "local":
|
||||
result.Local = &LocalEngineConfig{PathPrefix: c.PathPrefix}
|
||||
case "minio":
|
||||
mode := c.Mode
|
||||
if mode == "" {
|
||||
mode = "remote"
|
||||
}
|
||||
result.MinIO = &MinIOEngineConfig{Mode: mode, Endpoint: c.Endpoint, AccessKeyID: c.AccessKeyID, SecretAccessKey: c.SecretAccessKey, BucketName: c.BucketName, UseSSL: c.UseSSL, PathPrefix: c.PathPrefix}
|
||||
case "cos":
|
||||
result.COS = &COSEngineConfig{SecretID: c.AccessKeyID, SecretKey: c.SecretAccessKey, Region: c.Region, BucketName: c.BucketName, AppID: c.AppID, PathPrefix: c.PathPrefix, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
|
||||
case "tos":
|
||||
result.TOS = &TOSEngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
|
||||
case "s3":
|
||||
result.S3 = &S3EngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseSSL: c.UseSSL, ForcePathStyle: c.ForcePathStyle}
|
||||
case "oss":
|
||||
result.OSS = &OSSEngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseTempBucket: c.UseTempBucket, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
|
||||
case "ks3":
|
||||
result.KS3 = &KS3EngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix}
|
||||
case "obs":
|
||||
result.OBS = &OBSEngineConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKey: c.AccessKeyID, SecretKey: c.SecretAccessKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseSSL: c.UseSSL}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func NewStorageBackendResponse(backend *StorageBackend) StorageBackend {
|
||||
out := *backend
|
||||
out.Config = backend.Config.MaskSensitiveFields()
|
||||
return out
|
||||
}
|
||||
|
||||
// StorageBackendFromLegacy projects one provider entry from the old workspace
|
||||
// singleton JSON into the multi-instance model.
|
||||
func StorageBackendFromLegacy(tenantID uint64, provider string, legacy *StorageEngineConfig) *StorageBackend {
|
||||
if legacy == nil {
|
||||
return nil
|
||||
}
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
b := &StorageBackend{TenantID: tenantID, Provider: provider, Source: StorageBackendSourceUser, Status: StorageBackendStatusActive, LegacyAlias: true}
|
||||
switch provider {
|
||||
case "local":
|
||||
if legacy.Local == nil {
|
||||
return nil
|
||||
}
|
||||
b.Name, b.Config.PathPrefix = "Local", legacy.Local.PathPrefix
|
||||
case "minio":
|
||||
if legacy.MinIO == nil {
|
||||
return nil
|
||||
}
|
||||
c := legacy.MinIO
|
||||
b.Name = "MinIO"
|
||||
b.Config = StorageBackendConfig{Mode: c.Mode, Endpoint: c.Endpoint, AccessKeyID: c.AccessKeyID, SecretAccessKey: c.SecretAccessKey, BucketName: c.BucketName, UseSSL: c.UseSSL, PathPrefix: c.PathPrefix}
|
||||
case "cos":
|
||||
if legacy.COS == nil {
|
||||
return nil
|
||||
}
|
||||
c := legacy.COS
|
||||
b.Name = "COS"
|
||||
b.Config = StorageBackendConfig{Region: c.Region, AccessKeyID: c.SecretID, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, AppID: c.AppID, PathPrefix: c.PathPrefix, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
|
||||
case "tos":
|
||||
if legacy.TOS == nil {
|
||||
return nil
|
||||
}
|
||||
c := legacy.TOS
|
||||
b.Name = "TOS"
|
||||
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
|
||||
case "s3":
|
||||
if legacy.S3 == nil {
|
||||
return nil
|
||||
}
|
||||
c := legacy.S3
|
||||
b.Name = "S3"
|
||||
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseSSL: c.UseSSL, ForcePathStyle: c.ForcePathStyle}
|
||||
case "oss":
|
||||
if legacy.OSS == nil {
|
||||
return nil
|
||||
}
|
||||
c := legacy.OSS
|
||||
b.Name = "OSS"
|
||||
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseTempBucket: c.UseTempBucket, TempBucketName: c.TempBucketName, TempRegion: c.TempRegion}
|
||||
case "ks3":
|
||||
if legacy.KS3 == nil {
|
||||
return nil
|
||||
}
|
||||
c := legacy.KS3
|
||||
b.Name = "KS3"
|
||||
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix}
|
||||
case "obs":
|
||||
if legacy.OBS == nil {
|
||||
return nil
|
||||
}
|
||||
c := legacy.OBS
|
||||
b.Name = "OBS"
|
||||
b.Config = StorageBackendConfig{Endpoint: c.Endpoint, Region: c.Region, AccessKeyID: c.AccessKey, SecretAccessKey: c.SecretKey, BucketName: c.BucketName, PathPrefix: c.PathPrefix, UseSSL: c.UseSSL}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// StorageBackendFromEnvironment snapshots the process-wide storage backend for
|
||||
// a workspace. The row is read-only in the UI and keeps env-only deployments
|
||||
// on the same instance-resolution path as user-managed backends.
|
||||
func StorageBackendFromEnvironment(tenantID uint64) *StorageBackend {
|
||||
provider := strings.ToLower(strings.TrimSpace(os.Getenv("STORAGE_TYPE")))
|
||||
if provider == "" {
|
||||
provider = "local"
|
||||
}
|
||||
b := &StorageBackend{
|
||||
TenantID: tenantID, Name: "System " + strings.ToUpper(provider), Provider: provider,
|
||||
Source: StorageBackendSourceEnv, Status: StorageBackendStatusActive, LegacyAlias: true,
|
||||
}
|
||||
switch provider {
|
||||
case "local":
|
||||
b.Config.PathPrefix = strings.TrimSpace(os.Getenv("LOCAL_STORAGE_PATH_PREFIX"))
|
||||
case "minio":
|
||||
b.Config = StorageBackendConfig{Mode: "remote", Endpoint: os.Getenv("MINIO_ENDPOINT"), AccessKeyID: os.Getenv("MINIO_ACCESS_KEY_ID"), SecretAccessKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"), BucketName: os.Getenv("MINIO_BUCKET_NAME"), PathPrefix: os.Getenv("MINIO_PATH_PREFIX"), UseSSL: strings.EqualFold(os.Getenv("MINIO_USE_SSL"), "true")}
|
||||
case "cos":
|
||||
b.Config = StorageBackendConfig{Region: os.Getenv("COS_REGION"), AccessKeyID: os.Getenv("COS_SECRET_ID"), SecretAccessKey: os.Getenv("COS_SECRET_KEY"), BucketName: os.Getenv("COS_BUCKET_NAME"), AppID: os.Getenv("COS_APP_ID"), PathPrefix: os.Getenv("COS_PATH_PREFIX"), TempBucketName: os.Getenv("COS_TEMP_BUCKET_NAME"), TempRegion: os.Getenv("COS_TEMP_REGION")}
|
||||
case "tos":
|
||||
b.Config = StorageBackendConfig{Endpoint: os.Getenv("TOS_ENDPOINT"), Region: os.Getenv("TOS_REGION"), AccessKeyID: os.Getenv("TOS_ACCESS_KEY"), SecretAccessKey: os.Getenv("TOS_SECRET_KEY"), BucketName: os.Getenv("TOS_BUCKET_NAME"), PathPrefix: os.Getenv("TOS_PATH_PREFIX"), TempBucketName: os.Getenv("TOS_TEMP_BUCKET_NAME"), TempRegion: os.Getenv("TOS_TEMP_REGION")}
|
||||
case "s3":
|
||||
b.Config = StorageBackendConfig{Endpoint: os.Getenv("S3_ENDPOINT"), Region: os.Getenv("S3_REGION"), AccessKeyID: os.Getenv("S3_ACCESS_KEY"), SecretAccessKey: os.Getenv("S3_SECRET_KEY"), BucketName: os.Getenv("S3_BUCKET_NAME"), PathPrefix: os.Getenv("S3_PATH_PREFIX"), UseSSL: !strings.EqualFold(os.Getenv("S3_USE_SSL"), "false"), ForcePathStyle: strings.EqualFold(os.Getenv("S3_FORCE_PATH_STYLE"), "true")}
|
||||
case "oss":
|
||||
b.Config = StorageBackendConfig{Endpoint: os.Getenv("OSS_ENDPOINT"), Region: os.Getenv("OSS_REGION"), AccessKeyID: os.Getenv("OSS_ACCESS_KEY"), SecretAccessKey: os.Getenv("OSS_SECRET_KEY"), BucketName: os.Getenv("OSS_BUCKET_NAME"), PathPrefix: os.Getenv("OSS_PATH_PREFIX"), UseTempBucket: os.Getenv("OSS_TEMP_BUCKET_NAME") != "", TempBucketName: os.Getenv("OSS_TEMP_BUCKET_NAME"), TempRegion: os.Getenv("OSS_TEMP_REGION")}
|
||||
case "obs":
|
||||
b.Config = StorageBackendConfig{Endpoint: os.Getenv("OBS_ENDPOINT"), Region: os.Getenv("OBS_REGION"), AccessKeyID: os.Getenv("OBS_ACCESS_KEY"), SecretAccessKey: os.Getenv("OBS_SECRET_KEY"), BucketName: os.Getenv("OBS_BUCKET_NAME"), PathPrefix: os.Getenv("OBS_PATH_PREFIX"), UseSSL: !strings.EqualFold(os.Getenv("OBS_USE_SSL"), "false")}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestStorageBackendPathRoundTrip(t *testing.T) {
|
||||
path := BuildStorageBackendPath("backend-a", "cos://bucket/region/7/file.pdf")
|
||||
id, inner, ok := ParseStorageBackendPath(path)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "backend-a", id)
|
||||
assert.Equal(t, "cos://bucket/region/7/file.pdf", inner)
|
||||
assert.Equal(t, "cos", ParseProviderScheme(path))
|
||||
}
|
||||
|
||||
func TestSharesStorageBackendWithUsesConcreteInstance(t *testing.T) {
|
||||
aID, bID := "cos-a", "cos-b"
|
||||
a := &KnowledgeBase{StorageBackendID: &aID, StorageProviderConfig: &StorageProviderConfig{Provider: "cos"}}
|
||||
b := &KnowledgeBase{StorageBackendID: &bID, StorageProviderConfig: &StorageProviderConfig{Provider: "cos"}}
|
||||
assert.False(t, a.SharesStorageBackendWith(b, "", "cos"))
|
||||
|
||||
b.StorageBackendID = &aID
|
||||
assert.True(t, a.SharesStorageBackendWith(b, "", "cos"))
|
||||
}
|
||||
|
||||
func TestNewStorageBackendResponseMasksCredentials(t *testing.T) {
|
||||
backend := &StorageBackend{Config: StorageBackendConfig{AccessKeyID: "id", SecretAccessKey: "secret"}}
|
||||
response := NewStorageBackendResponse(backend)
|
||||
assert.Equal(t, RedactedSecretPlaceholder, response.Config.AccessKeyID)
|
||||
assert.Equal(t, RedactedSecretPlaceholder, response.Config.SecretAccessKey)
|
||||
assert.Equal(t, "id", backend.Config.AccessKeyID)
|
||||
}
|
||||
|
||||
func TestStorageBackendFromEnvironment(t *testing.T) {
|
||||
t.Setenv("STORAGE_TYPE", "s3")
|
||||
t.Setenv("S3_ENDPOINT", "https://s3.example.com")
|
||||
t.Setenv("S3_REGION", "ap-test-1")
|
||||
t.Setenv("S3_ACCESS_KEY", "access")
|
||||
t.Setenv("S3_SECRET_KEY", "secret")
|
||||
t.Setenv("S3_BUCKET_NAME", "bucket")
|
||||
|
||||
backend := StorageBackendFromEnvironment(42)
|
||||
require.NotNil(t, backend)
|
||||
assert.Equal(t, uint64(42), backend.TenantID)
|
||||
assert.Equal(t, "s3", backend.Provider)
|
||||
assert.Equal(t, StorageBackendSourceEnv, backend.Source)
|
||||
assert.True(t, backend.LegacyAlias)
|
||||
assert.Equal(t, "bucket", backend.Config.BucketName)
|
||||
}
|
||||
|
||||
func TestStorageBackendRejectsTraversingPathPrefix(t *testing.T) {
|
||||
backend := &StorageBackend{
|
||||
TenantID: 1, Name: "unsafe", Provider: "local",
|
||||
Config: StorageBackendConfig{PathPrefix: "../outside"},
|
||||
}
|
||||
require.Error(t, backend.Validate())
|
||||
}
|
||||
+18
-12
@@ -110,6 +110,8 @@ type Tenant struct {
|
||||
Credentials *CredentialsConfig `yaml:"credentials" json:"credentials" gorm:"type:jsonb"`
|
||||
// Storage engine config: parameters for Local, MinIO, COS. Used for document/file storage and docreader.
|
||||
StorageEngineConfig *StorageEngineConfig `yaml:"storage_engine_config" json:"storage_engine_config" gorm:"type:jsonb"`
|
||||
// DefaultStorageBackendID is the workspace default concrete storage instance.
|
||||
DefaultStorageBackendID *string `yaml:"default_storage_backend_id" json:"default_storage_backend_id,omitempty" gorm:"column:default_storage_backend_id;type:varchar(36)"`
|
||||
// Chat history config: knowledge base configuration for indexing and searching chat messages via vector search
|
||||
ChatHistoryConfig *ChatHistoryConfig `yaml:"chat_history_config" json:"chat_history_config" gorm:"type:jsonb"`
|
||||
// Retrieval config: global search/retrieval parameters shared by knowledge search and message search
|
||||
@@ -478,22 +480,26 @@ type MinIOEngineConfig struct {
|
||||
|
||||
// COSEngineConfig is for Tencent Cloud COS.
|
||||
type COSEngineConfig struct {
|
||||
SecretID string `json:"secret_id"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
Region string `json:"region"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
AppID string `json:"app_id"`
|
||||
PathPrefix string `json:"path_prefix"`
|
||||
SecretID string `json:"secret_id"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
Region string `json:"region"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
AppID string `json:"app_id"`
|
||||
PathPrefix string `json:"path_prefix"`
|
||||
TempBucketName string `json:"temp_bucket_name"`
|
||||
TempRegion string `json:"temp_region"`
|
||||
}
|
||||
|
||||
// TOSEngineConfig is for Volcengine TOS (火山引擎对象存储).
|
||||
type TOSEngineConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
PathPrefix string `json:"path_prefix"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
PathPrefix string `json:"path_prefix"`
|
||||
TempBucketName string `json:"temp_bucket_name"`
|
||||
TempRegion string `json:"temp_region"`
|
||||
}
|
||||
|
||||
// S3EngineConfig is for AWS S3 and S3-compatible object storage.
|
||||
|
||||
@@ -97,6 +97,14 @@ const (
|
||||
// APIKeyCapabilityManageVectorStores lets a key manage retrieval
|
||||
// infrastructure such as vector stores, parser engines, and storage checks.
|
||||
APIKeyCapabilityManageVectorStores APIKeyCapability = "manage_vector_stores"
|
||||
// APIKeyCapabilityManageStorageBackends lets a key manage object/file
|
||||
// storage backend instances (e.g. S3-compatible or local file storage):
|
||||
// their CRUD lifecycle, connectivity tests, and the tenant default
|
||||
// selection. It is separate from manage_vector_stores because storage
|
||||
// backends are the file-persistence layer (holding object-store
|
||||
// credentials and user-controllable endpoints) rather than retrieval
|
||||
// infrastructure.
|
||||
APIKeyCapabilityManageStorageBackends APIKeyCapability = "manage_storage_backends"
|
||||
// APIKeyCapabilityManageWebSearch lets a key manage tenant web-search
|
||||
// provider configurations and credentials.
|
||||
APIKeyCapabilityManageWebSearch APIKeyCapability = "manage_web_search"
|
||||
@@ -148,6 +156,8 @@ func NormalizeAPIKeyCapability(c APIKeyCapability) APIKeyCapability {
|
||||
return APIKeyCapabilityManageChannels
|
||||
case APIKeyCapabilityManageVectorStores:
|
||||
return APIKeyCapabilityManageVectorStores
|
||||
case APIKeyCapabilityManageStorageBackends:
|
||||
return APIKeyCapabilityManageStorageBackends
|
||||
case APIKeyCapabilityManageWebSearch:
|
||||
return APIKeyCapabilityManageWebSearch
|
||||
case APIKeyCapabilityRunEvaluations:
|
||||
|
||||
@@ -21,5 +21,6 @@ DROP TABLE IF EXISTS messages;
|
||||
DROP TABLE IF EXISTS sessions;
|
||||
DROP TABLE IF EXISTS knowledges;
|
||||
DROP TABLE IF EXISTS knowledge_bases;
|
||||
DROP TABLE IF EXISTS storage_backends;
|
||||
DROP TABLE IF EXISTS models;
|
||||
DROP TABLE IF EXISTS tenants;
|
||||
|
||||
@@ -15,6 +15,7 @@ CREATE TABLE IF NOT EXISTS tenants (
|
||||
web_search_config TEXT DEFAULT NULL,
|
||||
parser_engine_config TEXT DEFAULT NULL,
|
||||
storage_engine_config TEXT DEFAULT NULL,
|
||||
default_storage_backend_id VARCHAR(36),
|
||||
credentials TEXT DEFAULT NULL,
|
||||
chat_history_config TEXT,
|
||||
retrieval_config TEXT,
|
||||
@@ -69,6 +70,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
|
||||
pinned_at DATETIME NULL,
|
||||
asr_config TEXT,
|
||||
vector_store_id VARCHAR(36),
|
||||
storage_backend_id VARCHAR(36),
|
||||
creator_id VARCHAR(36),
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
@@ -78,6 +80,8 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
|
||||
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_id ON knowledge_bases(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_vector_store
|
||||
ON knowledge_bases(tenant_id, vector_store_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_storage_backend
|
||||
ON knowledge_bases(tenant_id, storage_backend_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_creator
|
||||
ON knowledge_bases(tenant_id, creator_id);
|
||||
|
||||
@@ -804,6 +808,25 @@ CREATE INDEX IF NOT EXISTS idx_vector_stores_tenant_id ON vector_stores(tenant_i
|
||||
CREATE INDEX IF NOT EXISTS idx_vector_stores_engine_type ON vector_stores(engine_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_vector_stores_deleted_at ON vector_stores(deleted_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS storage_backends (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
tenant_id INTEGER NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
provider VARCHAR(32) NOT NULL,
|
||||
config TEXT NOT NULL DEFAULT '{}',
|
||||
source VARCHAR(16) NOT NULL DEFAULT 'user',
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'active',
|
||||
legacy_alias BOOLEAN NOT NULL DEFAULT 0,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted_at DATETIME
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_name_tenant
|
||||
ON storage_backends(tenant_id, name) WHERE deleted_at IS NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_legacy_alias
|
||||
ON storage_backends(tenant_id, provider) WHERE deleted_at IS NULL AND legacy_alias = 1;
|
||||
CREATE INDEX IF NOT EXISTS idx_storage_backends_tenant ON storage_backends(tenant_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tenant_api_keys (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
tenant_id INTEGER NOT NULL,
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
DROP INDEX IF EXISTS idx_knowledge_bases_storage_backend;
|
||||
ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS storage_backend_id;
|
||||
ALTER TABLE tenants DROP COLUMN IF EXISTS default_storage_backend_id;
|
||||
DROP TABLE IF EXISTS storage_backends;
|
||||
@@ -0,0 +1,19 @@
|
||||
CREATE TABLE IF NOT EXISTS storage_backends (
|
||||
id VARCHAR(36) NOT NULL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
provider VARCHAR(32) NOT NULL,
|
||||
config JSONB NOT NULL DEFAULT '{}',
|
||||
source VARCHAR(16) NOT NULL DEFAULT 'user',
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'active',
|
||||
legacy_alias BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted_at TIMESTAMP NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_name_tenant ON storage_backends(tenant_id, name) WHERE deleted_at IS NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_legacy_alias ON storage_backends(tenant_id, provider) WHERE deleted_at IS NULL AND legacy_alias = TRUE;
|
||||
CREATE INDEX IF NOT EXISTS idx_storage_backends_tenant ON storage_backends(tenant_id);
|
||||
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS default_storage_backend_id VARCHAR(36);
|
||||
ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS storage_backend_id VARCHAR(36);
|
||||
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_storage_backend ON knowledge_bases(tenant_id, storage_backend_id);
|
||||
Reference in New Issue
Block a user