From 393ebf9f92ceda40939bd4117cbcb545918a945c Mon Sep 17 00:00:00 2001 From: wizardchen Date: Sat, 28 Feb 2026 17:39:59 +0800 Subject: [PATCH] feat: standardize storage configuration and enhance backward compatibility - Updated the `KnowledgeBase` struct to rename `cos_config` to `storage_config` for consistency across the application. - Implemented a custom `UnmarshalJSON` method to maintain backward compatibility with legacy responses that still use `cos_config`. - Modified various frontend components to reflect the new `storage_config` naming, ensuring seamless integration with the updated backend structure. - Enhanced Dockerfiles and service configurations to support the new storage engine setup. This update improves the clarity of storage configurations and ensures compatibility with existing data structures. --- client/knowledgebase.go | 22 ++++++++- docker/Dockerfile.app | 3 +- frontend/Dockerfile | 4 +- frontend/src/api/knowledge-base/index.ts | 2 +- .../knowledge/KnowledgeBaseEditorModal.vue | 7 +-- .../src/views/knowledge/KnowledgeBaseList.vue | 8 ++-- .../knowledge/settings/KBStorageSettings.vue | 9 +++- internal/application/service/file/local.go | 30 +++++++++++- .../application/service/image_multimodal.go | 8 +++- internal/application/service/knowledge.go | 48 +++++++++++++++++-- internal/handler/initialization.go | 13 +++++ internal/router/router.go | 18 +++++-- internal/types/knowledgebase.go | 21 +++++++- 13 files changed, 168 insertions(+), 25 deletions(-) diff --git a/client/knowledgebase.go b/client/knowledgebase.go index 10660a7f..3d16caeb 100644 --- a/client/knowledgebase.go +++ b/client/knowledgebase.go @@ -6,6 +6,7 @@ package client import ( "context" + "encoding/json" "fmt" "net/http" "time" @@ -25,7 +26,7 @@ type KnowledgeBase struct { EmbeddingModelID string `json:"embedding_model_id"` SummaryModelID string `json:"summary_model_id"` VLMConfig VLMConfig `json:"vlm_config"` - StorageConfig StorageConfig `json:"cos_config"` + StorageConfig StorageConfig `json:"storage_config"` ExtractConfig *ExtractConfig `json:"extract_config"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` @@ -99,6 +100,25 @@ type GraphRelation struct { Type string `json:"type"` } +// UnmarshalJSON keeps backward compatibility for legacy responses that still +// use `cos_config` instead of `storage_config`. +func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error { + type alias KnowledgeBase + aux := struct { + *alias + LegacyStorageConfig *StorageConfig `json:"cos_config"` + }{ + alias: (*alias)(kb), + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + if aux.LegacyStorageConfig != nil && kb.StorageConfig == (StorageConfig{}) { + kb.StorageConfig = *aux.LegacyStorageConfig + } + return nil +} + // KnowledgeBaseResponse knowledge base response type KnowledgeBaseResponse struct { Success bool `json:"success"` diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index 4ff37a81..b5466fe3 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -19,7 +19,7 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \ sed -i "s@deb.debian.org@${APK_MIRROR_ARG}@g" /etc/apt/sources.list.d/debian.sources; \ fi && \ apt-get update && \ - apt-get install -y git build-essential + apt-get install -y git build-essential libsqlite3-dev # Install migrate tool RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest @@ -63,6 +63,7 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \ apt-get update && \ apt-get install -y --no-install-recommends \ build-essential postgresql-client default-mysql-client ca-certificates tzdata sed curl bash vim wget \ + libsqlite3-0 \ python3 python3-pip python3-dev libffi-dev libssl-dev \ nodejs npm && \ python3 -m pip install --break-system-packages --upgrade pip setuptools wheel && \ diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 656d52cc..25d4f962 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,5 +1,5 @@ # 构建阶段 -FROM node:24-alpine as build-stage +FROM node:24-alpine AS build-stage WORKDIR /app @@ -22,7 +22,7 @@ COPY . . RUN pnpm run build # 生产阶段 -FROM nginx:stable-alpine as production-stage +FROM nginx:stable-alpine AS production-stage # 复制构建产物到nginx服务目录 COPY --from=build-stage /app/dist /usr/share/nginx/html diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index b75b278a..8be579b0 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -19,7 +19,7 @@ export function createKnowledgeBase(data: { enabled: boolean; model_id?: string; }; - cos_config?: any; + storage_config?: any; extract_config?: any; faq_config?: { index_mode: string; question_index_mode?: string }; }) { diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index 1b752e04..f46ebff0 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -131,6 +131,7 @@
@@ -374,7 +375,7 @@ const loadKBData = async () => { separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';'], parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined }, - storageProvider: (kb.cos_config?.provider || 'local') as string, + storageProvider: (kb.storage_config?.provider || 'local') as string, multimodalConfig: { enabled: !!kb.vlm_config?.enabled, vllmModelId: kb.vlm_config?.model_id || '' @@ -519,7 +520,7 @@ const buildSubmitData = () => { } // 存储引擎:仅传 provider,参数从全局设置读取 - data.cos_config = { + data.storage_config = { provider: formData.value.storageProvider || 'local' } @@ -607,7 +608,7 @@ const handleSubmit = async () => { multimodal: { enabled: !!data.vlm_config?.enabled }, - storageProvider: data.cos_config?.provider || 'local', + storageProvider: data.storage_config?.provider || 'local', nodeExtract: { enabled: data.extract_config?.enabled || false, text: data.extract_config?.text || '', diff --git a/frontend/src/views/knowledge/KnowledgeBaseList.vue b/frontend/src/views/knowledge/KnowledgeBaseList.vue index 7d2cb2ac..947a009e 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseList.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseList.vue @@ -143,7 +143,7 @@ - +
@@ -211,7 +211,7 @@
- +
@@ -308,7 +308,7 @@
- +
@@ -616,7 +616,7 @@ interface KB { showMore?: boolean; vlm_config?: { enabled?: boolean; model_id?: string }; extract_config?: { enabled?: boolean }; - cos_config?: { provider?: string; bucket_name?: string }; + storage_config?: { provider?: string; bucket_name?: string }; question_generation_config?: { enabled?: boolean; question_count?: number }; knowledge_count?: number; chunk_count?: number; diff --git a/frontend/src/views/knowledge/settings/KBStorageSettings.vue b/frontend/src/views/knowledge/settings/KBStorageSettings.vue index 85507b17..11102c0a 100644 --- a/frontend/src/views/knowledge/settings/KBStorageSettings.vue +++ b/frontend/src/views/knowledge/settings/KBStorageSettings.vue @@ -24,6 +24,7 @@ size="medium" placeholder="请选择存储引擎" style="width: 100%; min-width: 220px;" + :disabled="props.hasFiles" @change="handleChange" > -

{{ selectedOption.desc }}

+

知识库中已有文件,无法切换存储引擎。如需更换,请先清空知识库中的所有文件。

+

{{ selectedOption.desc }}

去全局设置中配置 @@ -55,6 +57,7 @@ import { useUIStore } from '@/stores/ui' const props = defineProps<{ storageProvider: string + hasFiles?: boolean }>() const emit = defineEmits<{ @@ -236,6 +239,10 @@ onMounted(load) color: #999; margin: 0; line-height: 1.4; + + &.locked-hint { + color: #e6a23c; + } } .go-settings { diff --git a/internal/application/service/file/local.go b/internal/application/service/file/local.go index 75563b8a..fe0640a7 100644 --- a/internal/application/service/file/local.go +++ b/internal/application/service/file/local.go @@ -7,6 +7,7 @@ import ( "mime/multipart" "os" "path/filepath" + "strings" "time" "github.com/Tencent/WeKnora/internal/logger" @@ -101,7 +102,8 @@ func (s *localFileService) SaveFile(ctx context.Context, func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { logger.Infof(ctx, "Getting file: %s", filePath) - resolved, err := secutils.SafePathUnderBase(s.baseDir, filePath) + candidate := s.normalizePathForBase(filePath) + resolved, err := secutils.SafePathUnderBase(s.baseDir, candidate) if err != nil { logger.Errorf(ctx, "Path traversal denied for GetFile: %v", err) return nil, fmt.Errorf("invalid file path: %w", err) @@ -123,7 +125,8 @@ func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.Rea func (s *localFileService) DeleteFile(ctx context.Context, filePath string) error { logger.Infof(ctx, "Deleting file: %s", filePath) - resolved, err := secutils.SafePathUnderBase(s.baseDir, filePath) + candidate := s.normalizePathForBase(filePath) + resolved, err := secutils.SafePathUnderBase(s.baseDir, candidate) if err != nil { logger.Errorf(ctx, "Path traversal denied for DeleteFile: %v", err) return fmt.Errorf("invalid file path: %w", err) @@ -180,3 +183,26 @@ func (s *localFileService) GetFileURL(ctx context.Context, filePath string) (str // Local storage doesn't support URLs, return the path return filePath, nil } + +// normalizePathForBase keeps backward compatibility for legacy file paths: +// - absolute path: "/data/files/tenant/.." +// - path under base dir: "tenant/.." +// - legacy relative with base prefix: "data/files/tenant/.." +func (s *localFileService) normalizePathForBase(filePath string) string { + clean := filepath.Clean(strings.TrimSpace(filePath)) + if clean == "." || clean == "" { + return clean + } + if filepath.IsAbs(clean) { + return clean + } + + // Strip duplicated base prefix in legacy relative paths, e.g. "data/files/..." + baseClean := filepath.Clean(s.baseDir) + baseNoSlash := strings.Trim(baseClean, string(filepath.Separator)) + cleanNoDot := strings.TrimPrefix(clean, "."+string(filepath.Separator)) + if strings.HasPrefix(cleanNoDot, baseNoSlash+string(filepath.Separator)) { + cleanNoDot = strings.TrimPrefix(cleanNoDot, baseNoSlash+string(filepath.Separator)) + } + return filepath.Join(baseClean, cleanNoDot) +} diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go index b2b686a5..6fcad570 100644 --- a/internal/application/service/image_multimodal.go +++ b/internal/application/service/image_multimodal.go @@ -92,7 +92,13 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(imgBytes)) } else { logger.Infof(ctx, "[ImageMultimodal] Image bytes read from %s, len=%d", payload.ImageLocalPath, len(imgBytes)) - defer os.Remove(payload.ImageLocalPath) + // Only clean up the local file when images have been re-uploaded to + // external object storage (COS/TOS) — the serving URL will be an + // absolute http(s) URL in that case. For local and MinIO storage the + // local file IS the serving file and must be kept. + if strings.HasPrefix(payload.ImageURL, "http://") || strings.HasPrefix(payload.ImageURL, "https://") { + defer os.Remove(payload.ImageLocalPath) + } } imageInfo := types.ImageInfo{ diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 830f4213..8e5f14b6 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -6756,6 +6756,8 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno } if hasKBFull { + logger.Infof(ctx, "[storage] buildStorageConfig use kb config: kb=%s provider=%s bucket=%s path_prefix=%s", + kb.ID, provider, sc.BucketName, sc.PathPrefix) return &types.DocParserStorageConfig{ Provider: strings.ToUpper(provider), Region: sc.Region, @@ -6809,6 +6811,8 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno } } + logger.Infof(ctx, "[storage] buildStorageConfig use merged tenant/global config: kb=%s provider=%s bucket=%s path_prefix=%s endpoint=%s", + kb.ID, strings.ToLower(out.Provider), out.BucketName, out.PathPrefix, out.Endpoint) return &out } @@ -6817,6 +6821,7 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno // Falls back to the global fileSvc when no tenant-level storage config is found. func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.KnowledgeBase) interfaces.FileService { if kb == nil { + logger.Infof(ctx, "[storage] resolveFileService fallback default: kb=nil") return s.fileSvc } @@ -6828,6 +6833,8 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno } if provider == "" || tenant == nil || tenant.StorageEngineConfig == nil { + logger.Infof(ctx, "[storage] resolveFileService fallback default: kb=%s provider=%q tenant_cfg=%v", + kb.ID, provider, tenant != nil && tenant.StorageEngineConfig != nil) return s.fileSvc } @@ -6837,10 +6844,28 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno switch provider { case "local": - baseDir := "/data/files" - if sec.Local != nil && sec.Local.PathPrefix != "" { - baseDir = sec.Local.PathPrefix + baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")) + if baseDir == "" { + baseDir = "/data/files" } + if sec.Local != nil && sec.Local.PathPrefix != "" { + // path_prefix is a subdirectory under local storage base dir, not a replacement. + // Validate it cannot escape baseDir (e.g. path traversal via ".."). + rawPrefix := strings.TrimSpace(sec.Local.PathPrefix) + prefix := strings.Trim(rawPrefix, "/\\") + if prefix != "" { + candidate := filepath.Join(baseDir, prefix) + safeBaseDir, pathErr := secutils.SafePathUnderBase(baseDir, candidate) + if pathErr != nil { + logger.Warnf(ctx, + "[storage] invalid local path_prefix ignored: kb=%s raw_prefix=%q err=%v", + kb.ID, rawPrefix, pathErr) + } else { + baseDir = safeBaseDir + } + } + } + logger.Infof(ctx, "[storage] resolveFileService local: kb=%s base_dir=%s", kb.ID, baseDir) return filesvc.NewLocalFileService(baseDir) case "minio": @@ -6886,6 +6911,7 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno logger.Errorf(ctx, "Failed to create %s file service from tenant config: %v, falling back to default", provider, err) return s.fileSvc } + logger.Infof(ctx, "[storage] resolveFileService selected: kb=%s provider=%s", kb.ID, provider) return svc } @@ -7275,13 +7301,27 @@ func (s *knowledgeService) convert( return nil, nil } + imageStorage := s.buildImageStorageConfig(ctx, kb) + if imageStorage == nil { + logger.Infof(ctx, "[storage] chunk image storage: kb=%s provider=%s image_storage=nil(use local temp/files)", + kb.ID, strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider))) + } else { + logger.Infof(ctx, "[storage] chunk image storage: kb=%s provider=%s bucket=%s endpoint=%s path_prefix=%s", + kb.ID, + imageStorage["provider"], + imageStorage["bucket_name"], + imageStorage["endpoint"], + imageStorage["path_prefix"], + ) + } + req := &types.ReadRequest{ URL: payload.URL, Title: knowledge.Title, ParserEngine: parserEngine, RequestID: payload.RequestId, ParserEngineOverrides: overrides, - ImageStorage: s.buildImageStorageConfig(ctx, kb), + ImageStorage: imageStorage, } if !isURL { diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go index 96ae30c1..e1d3395d 100644 --- a/internal/handler/initialization.go +++ b/internal/handler/initialization.go @@ -302,6 +302,19 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { if provider == "" { provider = "local" } + oldProvider := strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider)) + if oldProvider == "" { + oldProvider = "local" + } + if oldProvider != provider { + knowledgeList, err := h.knowledgeService.ListPagedKnowledgeByKnowledgeBaseID(ctx, + kbIdStr, &types.Pagination{Page: 1, PageSize: 1}, "", "", "") + if err == nil && knowledgeList != nil && knowledgeList.Total > 0 { + logger.Error(ctx, "Cannot change storage engine when files exist") + c.Error(errors.NewBadRequestError("知识库中已有文件,无法切换存储引擎")) + return + } + } kb.StorageConfig = types.StorageConfig{Provider: provider} // 更新知识图谱配置 diff --git a/internal/router/router.go b/internal/router/router.go index 740fa797..9d03a2de 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -619,18 +619,28 @@ func serveLocalFiles(r *gin.Engine) { logger.Infof(context.Background(), "[Router] Serving local storage files from %s at /files/", absDir) r.GET("/files/*filepath", func(c *gin.Context) { - reqPath := c.Param("filepath") - fullPath := filepath.Join(absDir, filepath.Clean(reqPath)) - - if !strings.HasPrefix(fullPath, absDir) { + reqPath := strings.TrimPrefix(c.Param("filepath"), "/") + cleanPath := filepath.Clean(reqPath) + if cleanPath == "." || strings.HasPrefix(cleanPath, "..") { + logger.Warnf(context.Background(), "[Router] Reject invalid /files path: raw=%q clean=%q", c.Param("filepath"), cleanPath) c.Status(http.StatusForbidden) return } + + fullPath := filepath.Join(absDir, cleanPath) + if !strings.HasPrefix(fullPath, absDir) { + logger.Warnf(context.Background(), "[Router] Reject escaped /files path: raw=%q full=%q base=%q", c.Param("filepath"), fullPath, absDir) + c.Status(http.StatusForbidden) + return + } + info, err := os.Stat(fullPath) if err != nil || info.IsDir() { + logger.Warnf(context.Background(), "[Router] /files not found: raw=%q full=%q err=%v", c.Param("filepath"), fullPath, err) c.Status(http.StatusNotFound) return } + logger.Infof(context.Background(), "[Router] /files hit: raw=%q full=%q", c.Param("filepath"), fullPath) c.File(fullPath) }) } diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index 1bf24fdc..d26212bc 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -60,7 +60,7 @@ type KnowledgeBase struct { // VLM config VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"` // Storage config - StorageConfig StorageConfig `yaml:"cos_config" json:"cos_config" gorm:"column:cos_config;type:json"` + StorageConfig StorageConfig `yaml:"storage_config" json:"storage_config" gorm:"column:cos_config;type:json"` // Extract config ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"` // FAQConfig stores FAQ specific configuration such as indexing strategy @@ -163,6 +163,25 @@ func (c *StorageConfig) Scan(value interface{}) error { return json.Unmarshal(b, c) } +// UnmarshalJSON keeps backward compatibility for legacy clients that still send +// `cos_config`, while standardizing the API field as `storage_config`. +func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error { + type alias KnowledgeBase + aux := struct { + *alias + LegacyStorageConfig *StorageConfig `json:"cos_config"` + }{ + alias: (*alias)(kb), + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + if aux.LegacyStorageConfig != nil && kb.StorageConfig == (StorageConfig{}) { + kb.StorageConfig = *aux.LegacyStorageConfig + } + return nil +} + // ImageProcessingConfig represents the image processing configuration type ImageProcessingConfig struct { // Model ID