feat: standardize storage configuration and enhance backward compatibility

- Updated the `KnowledgeBase` struct to rename `cos_config` to `storage_config` for consistency across the application.
- Implemented a custom `UnmarshalJSON` method to maintain backward compatibility with legacy responses that still use `cos_config`.
- Modified various frontend components to reflect the new `storage_config` naming, ensuring seamless integration with the updated backend structure.
- Enhanced Dockerfiles and service configurations to support the new storage engine setup.

This update improves the clarity of storage configurations and ensures compatibility with existing data structures.
This commit is contained in:
wizardchen
2026-02-28 17:39:59 +08:00
committed by lyingbug
parent 31755726a0
commit 393ebf9f92
13 changed files with 168 additions and 25 deletions
+21 -1
View File
@@ -6,6 +6,7 @@ package client
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
@@ -25,7 +26,7 @@ type KnowledgeBase struct {
EmbeddingModelID string `json:"embedding_model_id"`
SummaryModelID string `json:"summary_model_id"`
VLMConfig VLMConfig `json:"vlm_config"`
StorageConfig StorageConfig `json:"cos_config"`
StorageConfig StorageConfig `json:"storage_config"`
ExtractConfig *ExtractConfig `json:"extract_config"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
@@ -99,6 +100,25 @@ type GraphRelation struct {
Type string `json:"type"`
}
// UnmarshalJSON keeps backward compatibility for legacy responses that still
// use `cos_config` instead of `storage_config`.
func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error {
type alias KnowledgeBase
aux := struct {
*alias
LegacyStorageConfig *StorageConfig `json:"cos_config"`
}{
alias: (*alias)(kb),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
if aux.LegacyStorageConfig != nil && kb.StorageConfig == (StorageConfig{}) {
kb.StorageConfig = *aux.LegacyStorageConfig
}
return nil
}
// KnowledgeBaseResponse knowledge base response
type KnowledgeBaseResponse struct {
Success bool `json:"success"`
+2 -1
View File
@@ -19,7 +19,7 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \
sed -i "s@deb.debian.org@${APK_MIRROR_ARG}@g" /etc/apt/sources.list.d/debian.sources; \
fi && \
apt-get update && \
apt-get install -y git build-essential
apt-get install -y git build-essential libsqlite3-dev
# Install migrate tool
RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
@@ -63,6 +63,7 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \
apt-get update && \
apt-get install -y --no-install-recommends \
build-essential postgresql-client default-mysql-client ca-certificates tzdata sed curl bash vim wget \
libsqlite3-0 \
python3 python3-pip python3-dev libffi-dev libssl-dev \
nodejs npm && \
python3 -m pip install --break-system-packages --upgrade pip setuptools wheel && \
+2 -2
View File
@@ -1,5 +1,5 @@
# 构建阶段
FROM node:24-alpine as build-stage
FROM node:24-alpine AS build-stage
WORKDIR /app
@@ -22,7 +22,7 @@ COPY . .
RUN pnpm run build
# 生产阶段
FROM nginx:stable-alpine as production-stage
FROM nginx:stable-alpine AS production-stage
# 复制构建产物到nginx服务目录
COPY --from=build-stage /app/dist /usr/share/nginx/html
+1 -1
View File
@@ -19,7 +19,7 @@ export function createKnowledgeBase(data: {
enabled: boolean;
model_id?: string;
};
cos_config?: any;
storage_config?: any;
extract_config?: any;
faq_config?: { index_mode: string; question_index_mode?: string };
}) {
@@ -131,6 +131,7 @@
<div v-if="!isFAQ && formData" v-show="currentSection === 'storage'" class="section">
<KBStorageSettings
:storage-provider="formData.storageProvider"
:has-files="mode === 'edit' && hasFiles"
@update:storage-provider="handleStorageProviderUpdate"
/>
</div>
@@ -374,7 +375,7 @@ const loadKBData = async () => {
separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';'],
parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined
},
storageProvider: (kb.cos_config?.provider || 'local') as string,
storageProvider: (kb.storage_config?.provider || 'local') as string,
multimodalConfig: {
enabled: !!kb.vlm_config?.enabled,
vllmModelId: kb.vlm_config?.model_id || ''
@@ -519,7 +520,7 @@ const buildSubmitData = () => {
}
// 存储引擎:仅传 provider,参数从全局设置读取
data.cos_config = {
data.storage_config = {
provider: formData.value.storageProvider || 'local'
}
@@ -607,7 +608,7 @@ const handleSubmit = async () => {
multimodal: {
enabled: !!data.vlm_config?.enabled
},
storageProvider: data.cos_config?.provider || 'local',
storageProvider: data.storage_config?.provider || 'local',
nodeExtract: {
enabled: data.extract_config?.enabled || false,
text: data.extract_config?.text || '',
@@ -143,7 +143,7 @@
<t-icon name="relation" size="14px" />
</div>
</t-tooltip>
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.cos_config?.provider && kb.cos_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<t-tooltip v-if="kb.vlm_config?.enabled" :content="$t('knowledgeList.features.multimodal')" placement="top">
<div class="feature-badge multimodal">
<t-icon name="image" size="14px" />
</div>
@@ -211,7 +211,7 @@
<t-icon name="relation" size="14px" />
</div>
</t-tooltip>
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.cos_config?.provider && kb.cos_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.storage_config?.provider && kb.storage_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<div class="feature-badge multimodal">
<t-icon name="image" size="14px" />
</div>
@@ -308,7 +308,7 @@
<t-icon name="relation" size="14px" />
</div>
</t-tooltip>
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.cos_config?.provider && kb.cos_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.storage_config?.provider && kb.storage_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<div class="feature-badge multimodal">
<t-icon name="image" size="14px" />
</div>
@@ -616,7 +616,7 @@ interface KB {
showMore?: boolean;
vlm_config?: { enabled?: boolean; model_id?: string };
extract_config?: { enabled?: boolean };
cos_config?: { provider?: string; bucket_name?: string };
storage_config?: { provider?: string; bucket_name?: string };
question_generation_config?: { enabled?: boolean; question_count?: number };
knowledge_count?: number;
chunk_count?: number;
@@ -24,6 +24,7 @@
size="medium"
placeholder="请选择存储引擎"
style="width: 100%; min-width: 220px;"
:disabled="props.hasFiles"
@change="handleChange"
>
<t-option
@@ -40,7 +41,8 @@
</span>
</t-option>
</t-select>
<p v-if="selectedOption?.desc" class="option-hint">{{ selectedOption.desc }}</p>
<p v-if="props.hasFiles" class="option-hint locked-hint">知识库中已有文件,无法切换存储引擎。如需更换,请先清空知识库中的所有文件。</p>
<p v-else-if="selectedOption?.desc" class="option-hint">{{ selectedOption.desc }}</p>
<a v-if="showGoSettings" href="javascript:void(0)" class="go-settings" @click.prevent="goToStorageSettings">去全局设置中配置</a>
</div>
</div>
@@ -55,6 +57,7 @@ import { useUIStore } from '@/stores/ui'
const props = defineProps<{
storageProvider: string
hasFiles?: boolean
}>()
const emit = defineEmits<{
@@ -236,6 +239,10 @@ onMounted(load)
color: #999;
margin: 0;
line-height: 1.4;
&.locked-hint {
color: #e6a23c;
}
}
.go-settings {
+28 -2
View File
@@ -7,6 +7,7 @@ import (
"mime/multipart"
"os"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
@@ -101,7 +102,8 @@ func (s *localFileService) SaveFile(ctx context.Context,
func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
logger.Infof(ctx, "Getting file: %s", filePath)
resolved, err := secutils.SafePathUnderBase(s.baseDir, filePath)
candidate := s.normalizePathForBase(filePath)
resolved, err := secutils.SafePathUnderBase(s.baseDir, candidate)
if err != nil {
logger.Errorf(ctx, "Path traversal denied for GetFile: %v", err)
return nil, fmt.Errorf("invalid file path: %w", err)
@@ -123,7 +125,8 @@ func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.Rea
func (s *localFileService) DeleteFile(ctx context.Context, filePath string) error {
logger.Infof(ctx, "Deleting file: %s", filePath)
resolved, err := secutils.SafePathUnderBase(s.baseDir, filePath)
candidate := s.normalizePathForBase(filePath)
resolved, err := secutils.SafePathUnderBase(s.baseDir, candidate)
if err != nil {
logger.Errorf(ctx, "Path traversal denied for DeleteFile: %v", err)
return fmt.Errorf("invalid file path: %w", err)
@@ -180,3 +183,26 @@ func (s *localFileService) GetFileURL(ctx context.Context, filePath string) (str
// Local storage doesn't support URLs, return the path
return filePath, nil
}
// normalizePathForBase keeps backward compatibility for legacy file paths:
// - absolute path: "/data/files/tenant/.."
// - path under base dir: "tenant/.."
// - legacy relative with base prefix: "data/files/tenant/.."
func (s *localFileService) normalizePathForBase(filePath string) string {
clean := filepath.Clean(strings.TrimSpace(filePath))
if clean == "." || clean == "" {
return clean
}
if filepath.IsAbs(clean) {
return clean
}
// Strip duplicated base prefix in legacy relative paths, e.g. "data/files/..."
baseClean := filepath.Clean(s.baseDir)
baseNoSlash := strings.Trim(baseClean, string(filepath.Separator))
cleanNoDot := strings.TrimPrefix(clean, "."+string(filepath.Separator))
if strings.HasPrefix(cleanNoDot, baseNoSlash+string(filepath.Separator)) {
cleanNoDot = strings.TrimPrefix(cleanNoDot, baseNoSlash+string(filepath.Separator))
}
return filepath.Join(baseClean, cleanNoDot)
}
@@ -92,7 +92,13 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(imgBytes))
} else {
logger.Infof(ctx, "[ImageMultimodal] Image bytes read from %s, len=%d", payload.ImageLocalPath, len(imgBytes))
defer os.Remove(payload.ImageLocalPath)
// Only clean up the local file when images have been re-uploaded to
// external object storage (COS/TOS) — the serving URL will be an
// absolute http(s) URL in that case. For local and MinIO storage the
// local file IS the serving file and must be kept.
if strings.HasPrefix(payload.ImageURL, "http://") || strings.HasPrefix(payload.ImageURL, "https://") {
defer os.Remove(payload.ImageLocalPath)
}
}
imageInfo := types.ImageInfo{
+44 -4
View File
@@ -6756,6 +6756,8 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
}
if hasKBFull {
logger.Infof(ctx, "[storage] buildStorageConfig use kb config: kb=%s provider=%s bucket=%s path_prefix=%s",
kb.ID, provider, sc.BucketName, sc.PathPrefix)
return &types.DocParserStorageConfig{
Provider: strings.ToUpper(provider),
Region: sc.Region,
@@ -6809,6 +6811,8 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
}
}
logger.Infof(ctx, "[storage] buildStorageConfig use merged tenant/global config: kb=%s provider=%s bucket=%s path_prefix=%s endpoint=%s",
kb.ID, strings.ToLower(out.Provider), out.BucketName, out.PathPrefix, out.Endpoint)
return &out
}
@@ -6817,6 +6821,7 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
// Falls back to the global fileSvc when no tenant-level storage config is found.
func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.KnowledgeBase) interfaces.FileService {
if kb == nil {
logger.Infof(ctx, "[storage] resolveFileService fallback default: kb=nil")
return s.fileSvc
}
@@ -6828,6 +6833,8 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
}
if provider == "" || tenant == nil || tenant.StorageEngineConfig == nil {
logger.Infof(ctx, "[storage] resolveFileService fallback default: kb=%s provider=%q tenant_cfg=%v",
kb.ID, provider, tenant != nil && tenant.StorageEngineConfig != nil)
return s.fileSvc
}
@@ -6837,10 +6844,28 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
switch provider {
case "local":
baseDir := "/data/files"
if sec.Local != nil && sec.Local.PathPrefix != "" {
baseDir = sec.Local.PathPrefix
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
if baseDir == "" {
baseDir = "/data/files"
}
if sec.Local != nil && sec.Local.PathPrefix != "" {
// path_prefix is a subdirectory under local storage base dir, not a replacement.
// Validate it cannot escape baseDir (e.g. path traversal via "..").
rawPrefix := strings.TrimSpace(sec.Local.PathPrefix)
prefix := strings.Trim(rawPrefix, "/\\")
if prefix != "" {
candidate := filepath.Join(baseDir, prefix)
safeBaseDir, pathErr := secutils.SafePathUnderBase(baseDir, candidate)
if pathErr != nil {
logger.Warnf(ctx,
"[storage] invalid local path_prefix ignored: kb=%s raw_prefix=%q err=%v",
kb.ID, rawPrefix, pathErr)
} else {
baseDir = safeBaseDir
}
}
}
logger.Infof(ctx, "[storage] resolveFileService local: kb=%s base_dir=%s", kb.ID, baseDir)
return filesvc.NewLocalFileService(baseDir)
case "minio":
@@ -6886,6 +6911,7 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
logger.Errorf(ctx, "Failed to create %s file service from tenant config: %v, falling back to default", provider, err)
return s.fileSvc
}
logger.Infof(ctx, "[storage] resolveFileService selected: kb=%s provider=%s", kb.ID, provider)
return svc
}
@@ -7275,13 +7301,27 @@ func (s *knowledgeService) convert(
return nil, nil
}
imageStorage := s.buildImageStorageConfig(ctx, kb)
if imageStorage == nil {
logger.Infof(ctx, "[storage] chunk image storage: kb=%s provider=%s image_storage=nil(use local temp/files)",
kb.ID, strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider)))
} else {
logger.Infof(ctx, "[storage] chunk image storage: kb=%s provider=%s bucket=%s endpoint=%s path_prefix=%s",
kb.ID,
imageStorage["provider"],
imageStorage["bucket_name"],
imageStorage["endpoint"],
imageStorage["path_prefix"],
)
}
req := &types.ReadRequest{
URL: payload.URL,
Title: knowledge.Title,
ParserEngine: parserEngine,
RequestID: payload.RequestId,
ParserEngineOverrides: overrides,
ImageStorage: s.buildImageStorageConfig(ctx, kb),
ImageStorage: imageStorage,
}
if !isURL {
+13
View File
@@ -302,6 +302,19 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
if provider == "" {
provider = "local"
}
oldProvider := strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider))
if oldProvider == "" {
oldProvider = "local"
}
if oldProvider != provider {
knowledgeList, err := h.knowledgeService.ListPagedKnowledgeByKnowledgeBaseID(ctx,
kbIdStr, &types.Pagination{Page: 1, PageSize: 1}, "", "", "")
if err == nil && knowledgeList != nil && knowledgeList.Total > 0 {
logger.Error(ctx, "Cannot change storage engine when files exist")
c.Error(errors.NewBadRequestError("知识库中已有文件,无法切换存储引擎"))
return
}
}
kb.StorageConfig = types.StorageConfig{Provider: provider}
// 更新知识图谱配置
+14 -4
View File
@@ -619,18 +619,28 @@ func serveLocalFiles(r *gin.Engine) {
logger.Infof(context.Background(), "[Router] Serving local storage files from %s at /files/", absDir)
r.GET("/files/*filepath", func(c *gin.Context) {
reqPath := c.Param("filepath")
fullPath := filepath.Join(absDir, filepath.Clean(reqPath))
if !strings.HasPrefix(fullPath, absDir) {
reqPath := strings.TrimPrefix(c.Param("filepath"), "/")
cleanPath := filepath.Clean(reqPath)
if cleanPath == "." || strings.HasPrefix(cleanPath, "..") {
logger.Warnf(context.Background(), "[Router] Reject invalid /files path: raw=%q clean=%q", c.Param("filepath"), cleanPath)
c.Status(http.StatusForbidden)
return
}
fullPath := filepath.Join(absDir, cleanPath)
if !strings.HasPrefix(fullPath, absDir) {
logger.Warnf(context.Background(), "[Router] Reject escaped /files path: raw=%q full=%q base=%q", c.Param("filepath"), fullPath, absDir)
c.Status(http.StatusForbidden)
return
}
info, err := os.Stat(fullPath)
if err != nil || info.IsDir() {
logger.Warnf(context.Background(), "[Router] /files not found: raw=%q full=%q err=%v", c.Param("filepath"), fullPath, err)
c.Status(http.StatusNotFound)
return
}
logger.Infof(context.Background(), "[Router] /files hit: raw=%q full=%q", c.Param("filepath"), fullPath)
c.File(fullPath)
})
}
+20 -1
View File
@@ -60,7 +60,7 @@ type KnowledgeBase struct {
// VLM config
VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"`
// Storage config
StorageConfig StorageConfig `yaml:"cos_config" json:"cos_config" gorm:"column:cos_config;type:json"`
StorageConfig StorageConfig `yaml:"storage_config" json:"storage_config" gorm:"column:cos_config;type:json"`
// Extract config
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
// FAQConfig stores FAQ specific configuration such as indexing strategy
@@ -163,6 +163,25 @@ func (c *StorageConfig) Scan(value interface{}) error {
return json.Unmarshal(b, c)
}
// UnmarshalJSON keeps backward compatibility for legacy clients that still send
// `cos_config`, while standardizing the API field as `storage_config`.
func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error {
type alias KnowledgeBase
aux := struct {
*alias
LegacyStorageConfig *StorageConfig `json:"cos_config"`
}{
alias: (*alias)(kb),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
if aux.LegacyStorageConfig != nil && kb.StorageConfig == (StorageConfig{}) {
kb.StorageConfig = *aux.LegacyStorageConfig
}
return nil
}
// ImageProcessingConfig represents the image processing configuration
type ImageProcessingConfig struct {
// Model ID