feat(storage): add multi-instance storage backends per workspace

Introduce tenant-scoped storage backend registry so workspaces can register
multiple storage instances and bind each knowledge base to a specific backend,
with migration from legacy tenant-level storage config and new settings UI.
This commit is contained in:
wizardchen
2026-07-15 13:26:29 +08:00
committed by lyingbug
parent 139ec1e6b8
commit f8b3e5f148
58 changed files with 2863 additions and 470 deletions
@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
pinned_at DATETIME NULL,
asr_config TEXT,
vector_store_id VARCHAR(36),
storage_backend_id VARCHAR(36),
wiki_config TEXT,
indexing_strategy TEXT,
creator_id VARCHAR(36),
@@ -0,0 +1,58 @@
package repository
import (
"context"
"errors"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
)
type storageBackendRepository struct{ db *gorm.DB }
func NewStorageBackendRepository(db *gorm.DB) interfaces.StorageBackendRepository {
return &storageBackendRepository{db: db}
}
func (r *storageBackendRepository) Create(ctx context.Context, backend *types.StorageBackend) error {
return r.db.WithContext(ctx).Create(backend).Error
}
func (r *storageBackendRepository) GetByID(ctx context.Context, tenantID uint64, id string) (*types.StorageBackend, error) {
var backend types.StorageBackend
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).First(&backend).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return &backend, nil
}
func (r *storageBackendRepository) List(ctx context.Context, tenantID uint64) ([]*types.StorageBackend, error) {
var backends []*types.StorageBackend
err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).Order("created_at DESC").Find(&backends).Error
return backends, err
}
func (r *storageBackendRepository) Update(ctx context.Context, backend *types.StorageBackend) error {
return r.db.WithContext(ctx).Model(&types.StorageBackend{}).
Where("tenant_id = ? AND id = ?", backend.TenantID, backend.ID).
Select("name", "config", "status", "updated_at").Updates(backend).Error
}
func (r *storageBackendRepository) Delete(ctx context.Context, tenantID uint64, id string) error {
return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.StorageBackend{}).Error
}
func (r *storageBackendRepository) FindLegacyAlias(ctx context.Context, tenantID uint64, provider string) (*types.StorageBackend, error) {
var backend types.StorageBackend
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND provider = ? AND legacy_alias = ?", tenantID, provider, true).First(&backend).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return &backend, nil
}
@@ -100,6 +100,7 @@ type agentService struct {
webSearchStateService interfaces.WebSearchStateService
wikiPageService interfaces.WikiPageService
tenantService interfaces.TenantService
storageResolver interfaces.StorageBackendResolver
toolApprovalGate approval.MCPApproval
}
@@ -120,6 +121,7 @@ func NewAgentService(
webSearchStateService interfaces.WebSearchStateService,
wikiPageService interfaces.WikiPageService,
tenantService interfaces.TenantService,
storageResolver interfaces.StorageBackendResolver,
toolApprovalGate approval.MCPApproval,
) interfaces.AgentService {
return &agentService{
@@ -138,6 +140,7 @@ func NewAgentService(
webSearchStateService: webSearchStateService,
wikiPageService: wikiPageService,
tenantService: tenantService,
storageResolver: storageResolver,
toolApprovalGate: toolApprovalGate,
}
}
@@ -622,7 +625,7 @@ func (s *agentService) registerTools(
logger.Infof(ctx, "Registered web_fetch tool for session: %s", sessionID)
case tools.ToolDataAnalysis:
toolToRegister = tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, s.fileService, s.duckdb, sessionID)
toolToRegister = tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, s.fileService, s.duckdb, sessionID, s.storageResolver)
logger.Infof(ctx, "Registered data_analysis tool for session: %s", sessionID)
case tools.ToolDataSchema:
+19 -12
View File
@@ -11,7 +11,6 @@ import (
"github.com/Tencent/WeKnora/internal/agent/tools"
chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
@@ -406,6 +405,7 @@ type DataTableSummaryService struct {
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
sqlDB *sql.DB
storageResolver interfaces.StorageBackendResolver
}
// NewDataTableSummaryService creates a new DataTableSummaryService
@@ -419,6 +419,7 @@ func NewDataTableSummaryService(
retrieveEngine interfaces.RetrieveEngineRegistry,
ownership retriever.TenantStoreOwnership,
sqlDB *sql.DB,
storageResolver interfaces.StorageBackendResolver,
) interfaces.TaskHandler {
return &DataTableSummaryService{
modelService: modelService,
@@ -430,6 +431,7 @@ func NewDataTableSummaryService(
retrieveEngine: retrieveEngine,
ownership: ownership,
sqlDB: sqlDB,
storageResolver: storageResolver,
}
}
@@ -559,24 +561,29 @@ func (s *DataTableSummaryService) resolveFileServiceForKnowledge(ctx context.Con
if resources == nil || resources.knowledge == nil {
return s.fileService
}
if resources.tenant == nil || resources.tenant.StorageEngineConfig == nil {
if resources.tenant == nil {
return s.fileService
}
provider := types.InferStorageFromFilePath(resources.knowledge.FilePath)
if provider == "" {
if provider == "" && resources.tenant.StorageEngineConfig != nil {
provider = strings.ToLower(strings.TrimSpace(resources.tenant.StorageEngineConfig.DefaultProvider))
}
if provider == "" {
return s.fileService
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
resolvedSvc, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig(
provider,
resources.tenant.StorageEngineConfig,
baseDir,
)
backendID, _, _ := types.ParseStorageBackendPath(resources.knowledge.FilePath)
if backendID == "" && resources.knowledgeBase != nil && resources.knowledgeBase.StorageBackendID != nil {
backendID = strings.TrimSpace(*resources.knowledgeBase.StorageBackendID)
}
// New-model workspaces resolve via DefaultStorageBackendID even when no
// legacy StorageEngineConfig / provider is present, so gate on the resolver
// and a usable backendID/provider rather than requiring a non-empty provider.
if s.storageResolver == nil || (backendID == "" && provider == "") {
return s.fileService
}
resolvedSvc, resolvedProvider, err := s.storageResolver.ResolveFileService(ctx, resources.tenant, backendID, provider, baseDir)
if err != nil {
logger.Warnf(ctx, "[TableSummary] Failed to resolve file service for provider=%s, fallback to default: %v", provider, err)
return s.fileService
@@ -591,7 +598,7 @@ func (s *DataTableSummaryService) processTableData(ctx context.Context, resource
// 创建DuckDB会话并加载数据
sessionID := fmt.Sprintf("table_summary_%s", resources.knowledge.ID)
fileSvc := s.resolveFileServiceForKnowledge(ctx, resources)
duckdbTool := tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, fileSvc, s.sqlDB, sessionID)
duckdbTool := tools.NewDataAnalysisTool(s.knowledgeBaseService, s.knowledgeService, s.tenantService, fileSvc, s.sqlDB, sessionID, s.storageResolver)
defer duckdbTool.Cleanup(ctx)
// 使用knowledge.ID作为表名,根据文件类型自动加载数据
@@ -0,0 +1,113 @@
package file
import (
"context"
"fmt"
"io"
"mime/multipart"
"net/url"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// backendScopedFileService makes the storage instance part of every newly
// persisted path while delegating actual I/O to the existing provider driver.
type backendScopedFileService struct {
backendID string
inner interfaces.FileService
}
func NewBackendScopedFileService(backendID string, inner interfaces.FileService) interfaces.FileService {
return &backendScopedFileService{backendID: backendID, inner: inner}
}
func (s *backendScopedFileService) unwrap(path string) (string, error) {
id, inner, ok := types.ParseStorageBackendPath(path)
if !ok {
return path, nil
}
if id != s.backendID {
return "", fmt.Errorf("storage backend mismatch")
}
return inner, nil
}
func (s *backendScopedFileService) wrap(path string) string {
return types.BuildStorageBackendPath(s.backendID, path)
}
func (s *backendScopedFileService) CheckConnectivity(ctx context.Context) error {
return s.inner.CheckConnectivity(ctx)
}
func (s *backendScopedFileService) SaveFile(ctx context.Context, f *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
p, err := s.inner.SaveFile(ctx, f, tenantID, knowledgeID)
if err != nil {
return "", err
}
return s.wrap(p), nil
}
func (s *backendScopedFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, name string, temp bool) (string, error) {
p, err := s.inner.SaveBytes(ctx, data, tenantID, name, temp)
if err != nil {
return "", err
}
return s.wrap(p), nil
}
func (s *backendScopedFileService) GetFile(ctx context.Context, path string) (io.ReadCloser, error) {
p, err := s.unwrap(path)
if err != nil {
return nil, err
}
return s.inner.GetFile(ctx, p)
}
func (s *backendScopedFileService) GetFileURL(ctx context.Context, path string) (string, error) {
p, err := s.unwrap(path)
if err != nil {
return "", err
}
result, err := s.inner.GetFileURL(ctx, p)
if err != nil {
return "", err
}
scoped := s.wrap(p)
if result == p {
return scoped, nil
}
// Local storage may return an app-level presigned URL. Re-sign it with
// the scoped path so the proxy resolves the exact local instance instead
// of falling back to another backend of the same provider.
if u, parseErr := url.Parse(result); parseErr == nil && strings.HasSuffix(u.Path, "/api/v1/files/presigned") && u.Query().Get("file_path") == p {
basePath := strings.TrimSuffix(u.Path, "/api/v1/files/presigned")
baseURL := u.Scheme + "://" + u.Host + basePath
ttl := time.Duration(0)
if expires, convErr := strconv.ParseInt(u.Query().Get("expires"), 10, 64); convErr == nil {
ttl = time.Until(time.Unix(expires, 0))
}
if signed, signErr := secutils.SignFileURL(baseURL, scoped, secutils.ParseTenantIDFromStoragePath(scoped), ttl); signErr == nil {
return signed, nil
}
}
return result, nil
}
func (s *backendScopedFileService) DeleteFile(ctx context.Context, path string) error {
p, err := s.unwrap(path)
if err != nil {
return err
}
return s.inner.DeleteFile(ctx, p)
}
func (s *backendScopedFileService) CopyFile(ctx context.Context, path string, tenantID uint64, knowledgeID string) (string, error) {
p, err := s.unwrap(path)
if err != nil {
return "", err
}
result, err := s.inner.CopyFile(ctx, p, tenantID, knowledgeID)
if err != nil {
return "", err
}
return s.wrap(result), nil
}
@@ -0,0 +1,34 @@
package file
import (
"context"
"io"
"net/url"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBackendScopedLocalURLRetainsBackendID(t *testing.T) {
t.Setenv("SYSTEM_AES_KEY", "0123456789abcdef0123456789abcdef")
inner := NewLocalFileService(t.TempDir(), "https://weknora.example.com/base")
svc := NewBackendScopedFileService("backend-local-a", inner)
path, err := svc.SaveBytes(context.Background(), []byte("hello"), 7, "exports/image.txt", false)
require.NoError(t, err)
assert.Contains(t, path, "storage://backend-local-a/local://")
signed, err := svc.GetFileURL(context.Background(), path)
require.NoError(t, err)
u, err := url.Parse(signed)
require.NoError(t, err)
assert.Equal(t, path, u.Query().Get("file_path"))
reader, err := svc.GetFile(context.Background(), path)
require.NoError(t, err)
defer reader.Close()
data, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "hello", string(data))
}
+31 -9
View File
@@ -82,14 +82,14 @@ func NewFileServiceFromStorageConfig(
if pathPrefix == "" {
pathPrefix = "weknora"
}
svc, err := NewCosFileService(sec.COS.BucketName, sec.COS.Region, sec.COS.SecretID, sec.COS.SecretKey, pathPrefix)
svc, err := NewCosFileServiceWithTempBucket(sec.COS.BucketName, sec.COS.Region, sec.COS.SecretID, sec.COS.SecretKey, pathPrefix, sec.COS.TempBucketName, sec.COS.TempRegion)
return svc, p, err
case "tos":
if sec == nil || sec.TOS == nil || sec.TOS.Endpoint == "" || sec.TOS.Region == "" || sec.TOS.AccessKey == "" || sec.TOS.SecretKey == "" || sec.TOS.BucketName == "" {
return nil, p, fmt.Errorf("incomplete tos config")
}
svc, err := NewTosFileService(sec.TOS.Endpoint, sec.TOS.Region, sec.TOS.AccessKey, sec.TOS.SecretKey, sec.TOS.BucketName, sec.TOS.PathPrefix)
svc, err := NewTosFileServiceWithTempBucket(sec.TOS.Endpoint, sec.TOS.Region, sec.TOS.AccessKey, sec.TOS.SecretKey, sec.TOS.BucketName, sec.TOS.PathPrefix, sec.TOS.TempBucketName, sec.TOS.TempRegion)
return svc, p, err
case "s3":
if sec == nil || sec.S3 == nil || sec.S3.Endpoint == "" || sec.S3.Region == "" || sec.S3.AccessKey == "" || sec.S3.SecretKey == "" || sec.S3.BucketName == "" {
@@ -99,16 +99,38 @@ func NewFileServiceFromStorageConfig(
if pathPrefix == "" {
pathPrefix = "weknora/"
}
svc, err := NewS3FileService(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix)
svc, err := NewS3FileServiceWithOptions(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix, sec.S3.ForcePathStyle)
return svc, p, err
case "obs":
obsEndpoint := strings.TrimSpace(os.Getenv("OBS_ENDPOINT"))
obsRegion := strings.TrimSpace(os.Getenv("OBS_REGION"))
obsAccessKey := strings.TrimSpace(os.Getenv("OBS_ACCESS_KEY"))
obsSecretKey := strings.TrimSpace(os.Getenv("OBS_SECRET_KEY"))
obsBucketName := strings.TrimSpace(os.Getenv("OBS_BUCKET_NAME"))
obsPathPrefix := strings.TrimSpace(os.Getenv("OBS_PATH_PREFIX"))
obsEndpoint, obsRegion, obsAccessKey := "", "", ""
obsSecretKey, obsBucketName, obsPathPrefix := "", "", ""
if sec != nil && sec.OBS != nil {
obsEndpoint = strings.TrimSpace(sec.OBS.Endpoint)
obsRegion = strings.TrimSpace(sec.OBS.Region)
obsAccessKey = strings.TrimSpace(sec.OBS.AccessKey)
obsSecretKey = strings.TrimSpace(sec.OBS.SecretKey)
obsBucketName = strings.TrimSpace(sec.OBS.BucketName)
obsPathPrefix = strings.TrimSpace(sec.OBS.PathPrefix)
}
if obsEndpoint == "" {
obsEndpoint = strings.TrimSpace(os.Getenv("OBS_ENDPOINT"))
}
if obsRegion == "" {
obsRegion = strings.TrimSpace(os.Getenv("OBS_REGION"))
}
if obsAccessKey == "" {
obsAccessKey = strings.TrimSpace(os.Getenv("OBS_ACCESS_KEY"))
}
if obsSecretKey == "" {
obsSecretKey = strings.TrimSpace(os.Getenv("OBS_SECRET_KEY"))
}
if obsBucketName == "" {
obsBucketName = strings.TrimSpace(os.Getenv("OBS_BUCKET_NAME"))
}
if obsPathPrefix == "" {
obsPathPrefix = strings.TrimSpace(os.Getenv("OBS_PATH_PREFIX"))
}
if obsPathPrefix == "" {
obsPathPrefix = "weknora/"
}
+27 -4
View File
@@ -30,7 +30,7 @@ type s3FileService struct {
}
// newS3Client creates a bare s3FileService with just the SDK client initialised.
func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string) (*s3FileService, error) {
func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (*s3FileService, error) {
var cfg aws.Config
var err error
@@ -49,7 +49,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
// (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
var client *s3.Client
if endpoint != "" {
usePathStyle := !strings.Contains(endpoint, "amazonaws.com")
usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(endpoint)
o.UsePathStyle = usePathStyle
@@ -76,7 +76,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
func NewS3FileService(endpoint,
accessKey, secretKey, bucketName, region, pathPrefix string,
) (interfaces.FileService, error) {
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix)
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix, false)
if err != nil {
return nil, err
}
@@ -96,6 +96,25 @@ func NewS3FileService(endpoint,
return svc, nil
}
// NewS3FileServiceWithOptions is the instance-aware S3 constructor. Existing
// callers keep the historical endpoint-based path-style inference.
func NewS3FileServiceWithOptions(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (interfaces.FileService, error) {
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix, forcePathStyle)
if err != nil {
return nil, err
}
exists, err := svc.bucketExists(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to check bucket: %w", err)
}
if !exists {
if err = svc.createBucket(context.Background()); err != nil {
return nil, fmt.Errorf("failed to create bucket: %w", err)
}
}
return svc, nil
}
// bucketExists checks if the bucket exists
func (s *s3FileService) bucketExists(ctx context.Context) (bool, error) {
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
@@ -145,7 +164,11 @@ func (s *s3FileService) CheckConnectivity(ctx context.Context) error {
// CheckS3Connectivity tests S3 connectivity using the provided credentials.
// It creates a temporary service instance internally and delegates to CheckConnectivity.
func CheckS3Connectivity(ctx context.Context, endpoint, accessKey, secretKey, bucketName, region string) error {
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, "")
return CheckS3ConnectivityWithOptions(ctx, endpoint, accessKey, secretKey, bucketName, region, false)
}
func CheckS3ConnectivityWithOptions(ctx context.Context, endpoint, accessKey, secretKey, bucketName, region string, forcePathStyle bool) error {
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, "", forcePathStyle)
if err != nil {
return err
}
@@ -11,7 +11,6 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/application/repository"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
@@ -79,7 +78,8 @@ type ImageMultimodalService struct {
// (e.g. images were saved using the global MINIO_* env vars while the
// tenant's StorageEngineConfig.MinIO is empty). Mirrors the write-side
// fallback in knowledgeService.resolveFileService.
fileSvc interfaces.FileService
fileSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
// spanTracker records this image's subspan under the parent attempt's
// multimodal stage. nil-safe — falls back to no-op via tracker().
@@ -98,21 +98,23 @@ func NewImageMultimodalService(
taskEnqueuer interfaces.TaskEnqueuer,
redisClient *redis.Client,
fileSvc interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
spanTracker SpanTracker,
) interfaces.TaskHandler {
return &ImageMultimodalService{
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ownership: ownership,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
redisClient: redisClient,
fileSvc: fileSvc,
spanTracker: spanTracker,
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ownership: ownership,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
redisClient: redisClient,
fileSvc: fileSvc,
storageResolver: storageResolver,
spanTracker: spanTracker,
}
}
@@ -550,6 +552,7 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
return s.fileSvc
}
backendID, _, _ := types.ParseStorageBackendPath(payload.ImageURL)
provider := types.ParseProviderScheme(payload.ImageURL)
if provider == "" {
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
@@ -557,13 +560,20 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
logger.Warnf(ctx, "[ImageMultimodal] GetKnowledgeBaseByIDOnly failed: kb=%s err=%v", payload.KnowledgeBaseID, kbErr)
} else if kb != nil {
provider = strings.ToLower(strings.TrimSpace(kb.GetStorageProvider()))
if backendID == "" && kb.StorageBackendID != nil {
backendID = *kb.StorageBackendID
}
}
}
if s.storageResolver == nil {
return s.fileSvc
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
logger.Infof(ctx, "[ImageMultimodal] resolving file service: tenant=%d provider=%q LOCAL_STORAGE_BASE_DIR=%q imageURL=%s",
payload.TenantID, provider, baseDir, payload.ImageURL)
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
fileSvc, _, svcErr := s.storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir)
if svcErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] resolve file service failed (falling back to default): tenant=%d provider=%s err=%v",
payload.TenantID, provider, svcErr)
@@ -55,6 +55,7 @@ type knowledgeService struct {
tagRepo interfaces.KnowledgeTagRepository
tagService interfaces.KnowledgeTagService
fileSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
modelService interfaces.ModelService
task interfaces.TaskEnqueuer
taskInspector interfaces.TaskInspector
@@ -96,6 +97,7 @@ func NewKnowledgeService(
tagRepo interfaces.KnowledgeTagRepository,
tagService interfaces.KnowledgeTagService,
fileSvc interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
modelService interfaces.ModelService,
task interfaces.TaskEnqueuer,
taskInspector interfaces.TaskInspector,
@@ -122,6 +124,7 @@ func NewKnowledgeService(
tagRepo: tagRepo,
tagService: tagService,
fileSvc: fileSvc,
storageResolver: storageResolver,
modelService: modelService,
task: task,
taskInspector: taskInspector,
@@ -142,15 +142,18 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
}
switch provider {
case "cos":
concreteBackend := kb.StorageBackendID != nil && strings.TrimSpace(*kb.StorageBackendID) != ""
switch {
case concreteBackend:
// Registration already validated and tested the concrete instance.
case provider == "cos":
if tenant == nil || tenant.StorageEngineConfig == nil || tenant.StorageEngineConfig.COS == nil ||
tenant.StorageEngineConfig.COS.SecretID == "" || tenant.StorageEngineConfig.COS.SecretKey == "" ||
tenant.StorageEngineConfig.COS.Region == "" || tenant.StorageEngineConfig.COS.BucketName == "" {
logger.Error(ctx, "COS configuration incomplete for image multimodal processing")
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
}
case "minio":
case provider == "minio":
ok := false
if tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
m := tenant.StorageEngineConfig.MinIO
@@ -235,6 +235,11 @@ func mergeExtractConfig(base types.ExtractConfig, override *types.ExtractConfig)
}
func validateImageMultimodalConfig(ctx context.Context, kb *types.KnowledgeBase) error {
// Concrete backends are validated and connectivity-tested when registered.
// The checks below only apply to unmigrated provider-only bindings.
if kb != nil && kb.StorageBackendID != nil && strings.TrimSpace(*kb.StorageBackendID) != "" {
return nil
}
provider := kb.GetStorageProvider()
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
+50 -2
View File
@@ -115,6 +115,19 @@ func (s *knowledgeService) getVLMConfig(ctx context.Context, kb *types.Knowledge
func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.KnowledgeBase) *types.DocParserStorageConfig {
provider := kb.GetStorageProvider()
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
backendID := ""
if kb.StorageBackendID != nil {
backendID = *kb.StorageBackendID
}
if s.storageResolver != nil && tenant != nil {
if backend, err := s.storageResolver.ResolveBackend(ctx, tenant, backendID, provider); err == nil && backend != nil {
provider = backend.Provider
tenantCopy := *tenant
tenantCopy.StorageEngineConfig = backend.ToStorageEngineConfig()
tenant = &tenantCopy
}
}
if provider == "" {
provider = "local"
}
@@ -131,7 +144,7 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
hasKBFull = sc.SecretID != "" && sc.BucketName != ""
case "minio":
hasKBFull = sc.BucketName != ""
case "local", "tos", "s3", "oss", "ks3":
case "local", "tos", "s3", "oss", "ks3", "obs":
hasKBFull = false
}
@@ -153,7 +166,6 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
var out types.DocParserStorageConfig
out.Provider = strings.ToUpper(provider)
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant != nil && tenant.StorageEngineConfig != nil {
sec := tenant.StorageEngineConfig
if sec.DefaultProvider != "" && provider == "" {
@@ -228,6 +240,15 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
out.BucketName = sec.KS3.BucketName
out.PathPrefix = sec.KS3.PathPrefix
}
case "obs":
if sec.OBS != nil {
out.Endpoint = sec.OBS.Endpoint
out.Region = sec.OBS.Region
out.AccessKeyID = sec.OBS.AccessKey
out.SecretAccessKey = sec.OBS.SecretKey
out.BucketName = sec.OBS.BucketName
out.PathPrefix = sec.OBS.PathPrefix
}
}
}
@@ -248,6 +269,21 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
provider := kb.GetStorageProvider()
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
backendID := ""
if kb.StorageBackendID != nil {
backendID = strings.TrimSpace(*kb.StorageBackendID)
}
if s.storageResolver != nil && tenant != nil {
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
svc, resolvedProvider, err := s.storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir)
if err == nil && svc != nil {
logger.Infof(ctx, "[storage] resolveFileService selected instance: kb=%s backend=%s provider=%s", kb.ID, backendID, resolvedProvider)
return svc
}
if err != nil {
logger.Errorf(ctx, "Failed to resolve storage backend for kb=%s: %v", kb.ID, err)
}
}
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
}
@@ -274,6 +310,18 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
// the provider inferred from the file path. This protects historical data when
// tenant/KB config changes but files were stored under the old provider.
func (s *knowledgeService) resolveFileServiceForPath(ctx context.Context, kb *types.KnowledgeBase, filePath string) interfaces.FileService {
if backendID, inner, ok := types.ParseStorageBackendPath(filePath); ok && s.storageResolver != nil {
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant != nil {
provider := types.ParseProviderScheme(inner)
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
if resolved, _, err := s.storageResolver.ResolveFileService(ctx, tenant, backendID, provider, baseDir); err == nil {
return resolved
} else {
logger.Warnf(ctx, "[storage] failed to resolve backend from file path: backend=%s err=%v", backendID, err)
}
}
}
svc := s.resolveFileService(ctx, kb)
if filePath == "" {
return svc
+81 -46
View File
@@ -27,21 +27,22 @@ var ErrInvalidTenantID = errors.New("invalid tenant ID")
// knowledgeBaseService implements the knowledge base service interface
type knowledgeBaseService struct {
repo interfaces.KnowledgeBaseRepository
kgRepo interfaces.KnowledgeRepository
chunkRepo interfaces.ChunkRepository
shareRepo interfaces.KBShareRepository
kbShareService interfaces.KBShareService
modelService interfaces.ModelService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
tenantRepo interfaces.TenantRepository
fileSvc interfaces.FileService
graphEngine interfaces.RetrieveGraphRepository
asynqClient interfaces.TaskEnqueuer
dsRepo interfaces.DataSourceRepository
syncLogRepo interfaces.SyncLogRepository
dsScheduler *datasource.Scheduler
repo interfaces.KnowledgeBaseRepository
kgRepo interfaces.KnowledgeRepository
chunkRepo interfaces.ChunkRepository
shareRepo interfaces.KBShareRepository
kbShareService interfaces.KBShareService
modelService interfaces.ModelService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
tenantRepo interfaces.TenantRepository
fileSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
graphEngine interfaces.RetrieveGraphRepository
asynqClient interfaces.TaskEnqueuer
dsRepo interfaces.DataSourceRepository
syncLogRepo interfaces.SyncLogRepository
dsScheduler *datasource.Scheduler
}
// NewKnowledgeBaseService creates a new knowledge base service
@@ -55,6 +56,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
ownership retriever.TenantStoreOwnership,
tenantRepo interfaces.TenantRepository,
fileSvc interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
graphEngine interfaces.RetrieveGraphRepository,
asynqClient interfaces.TaskEnqueuer,
dsRepo interfaces.DataSourceRepository,
@@ -62,21 +64,22 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
dsScheduler *datasource.Scheduler,
) interfaces.KnowledgeBaseService {
return &knowledgeBaseService{
repo: repo,
kgRepo: kgRepo,
chunkRepo: chunkRepo,
shareRepo: shareRepo,
kbShareService: kbShareService,
modelService: modelService,
retrieveEngine: retrieveEngine,
ownership: ownership,
tenantRepo: tenantRepo,
fileSvc: fileSvc,
graphEngine: graphEngine,
asynqClient: asynqClient,
dsRepo: dsRepo,
syncLogRepo: syncLogRepo,
dsScheduler: dsScheduler,
repo: repo,
kgRepo: kgRepo,
chunkRepo: chunkRepo,
shareRepo: shareRepo,
kbShareService: kbShareService,
modelService: modelService,
retrieveEngine: retrieveEngine,
ownership: ownership,
tenantRepo: tenantRepo,
fileSvc: fileSvc,
storageResolver: storageResolver,
graphEngine: graphEngine,
asynqClient: asynqClient,
dsRepo: dsRepo,
syncLogRepo: syncLogRepo,
dsScheduler: dsScheduler,
}
}
@@ -119,6 +122,9 @@ func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
}
kb.EnsureDefaults()
applyTenantDefaultStorageProvider(ctx, kb)
if err := s.applyAndValidateStorageBackend(ctx, kb); err != nil {
return nil, err
}
// Fold empty-string vector_store_id into nil so this path and the
// retrieve-engine factory's pre-condition share a single representation.
@@ -150,6 +156,37 @@ func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
return kb, nil
}
func (s *knowledgeBaseService) applyAndValidateStorageBackend(ctx context.Context, kb *types.KnowledgeBase) error {
if s.storageResolver == nil || kb == nil {
return nil
}
tenant, _ := types.TenantInfoFromContext(ctx)
if tenant == nil {
return apperrors.NewBadRequestError("workspace context missing")
}
id := ""
if kb.StorageBackendID != nil {
id = strings.TrimSpace(*kb.StorageBackendID)
}
provider := kb.GetStorageProvider()
// A newly created KB without an explicit instance follows the concrete
// tenant default. The legacy provider is only a fallback for workspaces
// that have not been migrated yet.
if id == "" && tenant.DefaultStorageBackendID != nil && strings.TrimSpace(*tenant.DefaultStorageBackendID) != "" {
provider = ""
}
backend, err := s.storageResolver.ResolveBackend(ctx, tenant, id, provider)
if err != nil {
return apperrors.NewBadRequestError("storage backend is unavailable").WithDetails(err.Error())
}
if backend == nil {
return nil
}
kb.StorageBackendID = &backend.ID
kb.SetStorageProvider(backend.Provider)
return nil
}
// applyTenantDefaultStorageProvider fills an empty KB storage provider from the
// tenant's global default (Settings → Storage engine). Frontend should send the
// same value; this keeps API clients and legacy UIs consistent.
@@ -978,22 +1015,19 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context,
"cross-store cloning is not yet supported")
}
// Defense 3: storage backend must match — only meaningful when the
// tenant has a StorageEngineConfig. Without it, resolveFileService
// ignores per-KB provider pins and routes ALL KBs to the global
// storage service, so a clone can never span two real backends and
// the pins must NOT be used to reject (that would be a false positive).
// When a tenant config exists, pins are honored, so compare effective
// providers and reject a genuine cross-backend clone up front (it would
// otherwise fail mid-clone with ErrCrossBackendCopy).
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil && tenant.StorageEngineConfig != nil {
tenantDefault := tenant.StorageEngineConfig.DefaultProvider
srcProvider := sourceKB.EffectiveStorageProvider(tenantDefault)
dstProvider := targetKB.EffectiveStorageProvider(tenantDefault)
if srcProvider != "" && dstProvider != "" && srcProvider != dstProvider {
return nil, nil, apperrors.NewBadRequestError(fmt.Sprintf(
"source and target knowledge bases use different storage backends (%s vs %s); "+
"cross-storage-backend cloning is not supported", srcProvider, dstProvider))
// Defense 3: the concrete storage instance must match. Comparing only
// provider names would incorrectly allow COS-A -> COS-B clones.
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil {
defaultID, defaultProvider := "", ""
if tenant.DefaultStorageBackendID != nil {
defaultID = *tenant.DefaultStorageBackendID
}
if tenant.StorageEngineConfig != nil {
defaultProvider = tenant.StorageEngineConfig.DefaultProvider
}
if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) {
return nil, nil, apperrors.NewBadRequestError(
"source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported")
}
}
} else {
@@ -1016,6 +1050,7 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context,
SummaryModelID: sourceKB.SummaryModelID,
VLMConfig: sourceKB.VLMConfig,
StorageProviderConfig: sourceKB.StorageProviderConfig,
StorageBackendID: sourceKB.StorageBackendID,
StorageConfig: sourceKB.StorageConfig,
FAQConfig: faqConfig,
VectorStoreID: sourceKB.VectorStoreID,
@@ -0,0 +1,280 @@
package service
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
apperrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type StorageBackendService struct {
repo interfaces.StorageBackendRepository
db *gorm.DB
}
func NewStorageBackendService(repo interfaces.StorageBackendRepository, db *gorm.DB) *StorageBackendService {
return &StorageBackendService{repo: repo, db: db}
}
func (s *StorageBackendService) Create(ctx context.Context, backend *types.StorageBackend) error {
if err := backend.Validate(); err != nil {
return err
}
if err := validateStorageBackendEndpoint(backend); err != nil {
return err
}
if err := s.Test(ctx, backend); err != nil {
return apperrors.NewBadRequestError("storage connection test failed").WithDetails(err.Error())
}
backend.CreatedAt, backend.UpdatedAt = time.Now(), time.Now()
if err := s.repo.Create(ctx, backend); err != nil {
if strings.Contains(strings.ToLower(err.Error()), "unique") {
return apperrors.NewConflictError("a storage backend with this name already exists")
}
return err
}
return nil
}
func (s *StorageBackendService) Update(ctx context.Context, incoming *types.StorageBackend) error {
existing, err := s.repo.GetByID(ctx, incoming.TenantID, incoming.ID)
if err != nil {
return err
}
if existing == nil {
return apperrors.NewNotFoundError("storage backend not found")
}
if existing.Source == types.StorageBackendSourceEnv {
return apperrors.NewBadRequestError("environment storage backend is read-only")
}
incoming.Provider = existing.Provider
incoming.Config = incoming.Config.MergeSecrets(existing.Config)
if incoming.Config.LocationKey(existing.Provider) != existing.Config.LocationKey(existing.Provider) {
return apperrors.NewBadRequestError("endpoint, region, bucket and path prefix are immutable; use storage migration instead")
}
if incoming.Status == "" {
incoming.Status = existing.Status
}
if incoming.Status == types.StorageBackendStatusDisabled && existing.Status != types.StorageBackendStatusDisabled {
var references int64
if err := s.db.WithContext(ctx).Model(&types.Tenant{}).Where("id = ? AND default_storage_backend_id = ?", incoming.TenantID, incoming.ID).Count(&references).Error; err != nil {
return err
}
if references == 0 {
if err := s.db.WithContext(ctx).Model(&types.KnowledgeBase{}).Where("tenant_id = ? AND storage_backend_id = ?", incoming.TenantID, incoming.ID).Count(&references).Error; err != nil {
return err
}
}
if references > 0 {
return apperrors.NewBadRequestError("a default or bound storage backend cannot be disabled")
}
}
if err := incoming.Validate(); err != nil {
return err
}
if err := validateStorageBackendEndpoint(incoming); err != nil {
return err
}
if err := s.Test(ctx, incoming); err != nil {
return apperrors.NewBadRequestError("storage connection test failed").WithDetails(err.Error())
}
incoming.UpdatedAt = time.Now()
return s.repo.Update(ctx, incoming)
}
func (s *StorageBackendService) Delete(ctx context.Context, tenantID uint64, id string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var backend types.StorageBackend
query := tx.Where("tenant_id = ? AND id = ?", tenantID, id)
if tx.Dialector.Name() == "postgres" {
query = query.Clauses(clause.Locking{Strength: "UPDATE"})
}
if err := query.First(&backend).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return apperrors.NewNotFoundError("storage backend not found")
}
return err
}
if backend.Source == types.StorageBackendSourceEnv {
return apperrors.NewBadRequestError("environment storage backend is read-only")
}
var defaultCount int64
if err := tx.Model(&types.Tenant{}).Where("id = ? AND default_storage_backend_id = ?", tenantID, id).Count(&defaultCount).Error; err != nil {
return err
}
if defaultCount > 0 {
return apperrors.NewBadRequestError("default storage backend cannot be deleted")
}
var kbCount int64
if err := tx.Model(&types.KnowledgeBase{}).Where("tenant_id = ? AND storage_backend_id = ?", tenantID, id).Count(&kbCount).Error; err != nil {
return err
}
if kbCount > 0 {
return apperrors.NewBadRequestError(fmt.Sprintf("storage backend still has %d knowledge base(s) bound to it", kbCount))
}
if backend.LegacyAlias {
return apperrors.NewBadRequestError("legacy storage backend cannot be deleted while old file paths may reference it")
}
return tx.Delete(&backend).Error
})
}
func (s *StorageBackendService) SetDefault(ctx context.Context, tenantID uint64, id string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var backend types.StorageBackend
query := tx.Where("tenant_id = ? AND id = ?", tenantID, id)
if tx.Dialector.Name() == "postgres" {
query = query.Clauses(clause.Locking{Strength: "UPDATE"})
}
if err := query.First(&backend).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return apperrors.NewNotFoundError("storage backend not found")
}
return err
}
if backend.Status != types.StorageBackendStatusActive {
return apperrors.NewBadRequestError("only an active storage backend can be the default")
}
return tx.Model(&types.Tenant{}).Where("id = ?", tenantID).Update("default_storage_backend_id", id).Error
})
}
func (s *StorageBackendService) Test(ctx context.Context, backend *types.StorageBackend) error {
if err := backend.Validate(); err != nil {
return err
}
if err := validateStorageBackendEndpoint(backend); err != nil {
return err
}
if backend.Provider == "local" {
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
if baseDir == "" {
baseDir = "/data/files"
}
candidate := filepath.Join(baseDir, strings.Trim(strings.TrimSpace(backend.Config.PathPrefix), "/\\"))
safeDir, err := secutils.SafePathUnderBase(baseDir, candidate)
if err != nil {
return err
}
if err := os.MkdirAll(safeDir, 0o755); err != nil {
return fmt.Errorf("create local storage directory: %w", err)
}
}
c := backend.Config
switch backend.Provider {
case "local":
fileService, _, err := filesvc.NewFileServiceFromStorageConfig("local", backend.ToStorageEngineConfig(), "")
if err != nil {
return err
}
return fileService.CheckConnectivity(ctx)
case "minio":
if c.Mode == "docker" {
c.Endpoint = os.Getenv("MINIO_ENDPOINT")
c.AccessKeyID = os.Getenv("MINIO_ACCESS_KEY_ID")
c.SecretAccessKey = os.Getenv("MINIO_SECRET_ACCESS_KEY")
if c.BucketName == "" {
c.BucketName = os.Getenv("MINIO_BUCKET_NAME")
}
}
return filesvc.CheckMinioConnectivity(ctx, c.Endpoint, c.AccessKeyID, c.SecretAccessKey, c.BucketName, c.UseSSL)
case "cos":
return filesvc.CheckCosConnectivity(ctx, c.BucketName, c.Region, c.AccessKeyID, c.SecretAccessKey)
case "tos":
return filesvc.CheckTosConnectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
case "s3":
return filesvc.CheckS3ConnectivityWithOptions(ctx, c.Endpoint, c.AccessKeyID, c.SecretAccessKey, c.BucketName, c.Region, c.ForcePathStyle)
case "oss":
return filesvc.CheckOssConnectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
case "ks3":
return filesvc.CheckKS3Connectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
case "obs":
return filesvc.CheckObsConnectivity(ctx, c.Endpoint, c.Region, c.AccessKeyID, c.SecretAccessKey, c.BucketName)
default:
return fmt.Errorf("unsupported storage provider: %s", backend.Provider)
}
}
func (s *StorageBackendService) ResolveBackend(ctx context.Context, tenant *types.Tenant, backendID, provider string) (*types.StorageBackend, error) {
if tenant == nil {
return nil, fmt.Errorf("workspace context missing")
}
backendID = strings.TrimSpace(backendID)
provider = strings.ToLower(strings.TrimSpace(provider))
if backendID == "" && provider != "" {
backend, err := s.repo.FindLegacyAlias(ctx, tenant.ID, provider)
if err != nil || backend != nil {
return backend, err
}
}
if backendID == "" && tenant.DefaultStorageBackendID != nil {
backendID = strings.TrimSpace(*tenant.DefaultStorageBackendID)
}
if backendID != "" {
backend, err := s.repo.GetByID(ctx, tenant.ID, backendID)
if err != nil {
return nil, err
}
if backend == nil {
return nil, fmt.Errorf("storage backend not found")
}
if backend.Status != types.StorageBackendStatusActive {
return nil, fmt.Errorf("storage backend is not active")
}
return backend, nil
}
return nil, nil
}
func (s *StorageBackendService) ResolveFileService(ctx context.Context, tenant *types.Tenant, backendID, provider, localBaseDir string) (interfaces.FileService, string, error) {
backend, err := s.ResolveBackend(ctx, tenant, backendID, provider)
if err != nil {
return nil, "", err
}
if backend != nil {
inner, provider, err := filesvc.NewFileServiceFromStorageConfig(backend.Provider, backend.ToStorageEngineConfig(), localBaseDir)
if err != nil {
return nil, provider, err
}
return filesvc.NewBackendScopedFileService(backend.ID, inner), provider, nil
}
if tenant == nil {
return nil, "", fmt.Errorf("workspace context missing")
}
return filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, localBaseDir)
}
func validateStorageBackendEndpoint(backend *types.StorageBackend) error {
if backend.Provider == "local" || (backend.Provider == "minio" && backend.Config.Mode == "docker") {
return nil
}
endpoint := strings.TrimSpace(backend.Config.Endpoint)
if backend.Provider == "cos" || endpoint == "" {
return nil
}
if !strings.Contains(endpoint, "://") {
scheme := "https://"
if backend.Provider == "minio" && !backend.Config.UseSSL {
scheme = "http://"
}
endpoint = scheme + endpoint
}
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
return apperrors.NewBadRequestError("storage endpoint failed SSRF validation").WithDetails(err.Error())
}
return nil
}
var _ interfaces.StorageBackendService = (*StorageBackendService)(nil)
var _ interfaces.StorageBackendResolver = (*StorageBackendService)(nil)
@@ -0,0 +1,42 @@
package service_test
import (
"context"
"io"
"testing"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func TestStorageBackendResolverScopesPathsAndTenant(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&types.StorageBackend{}))
repo := repository.NewStorageBackendRepository(db)
backend := &types.StorageBackend{TenantID: 7, Name: "Local A", Provider: "local", Config: types.StorageBackendConfig{}, LegacyAlias: true}
require.NoError(t, repo.Create(context.Background(), backend))
resolver := service.NewStorageBackendService(repo, db)
svc, provider, err := resolver.ResolveFileService(context.Background(), &types.Tenant{ID: 7}, backend.ID, "local", t.TempDir())
require.NoError(t, err)
assert.Equal(t, "local", provider)
path, err := svc.SaveBytes(context.Background(), []byte("scoped"), 7, "exports/a.txt", false)
require.NoError(t, err)
assert.Contains(t, path, "storage://"+backend.ID+"/local://")
reader, err := svc.GetFile(context.Background(), path)
require.NoError(t, err)
defer reader.Close()
data, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "scoped", string(data))
_, _, err = resolver.ResolveFileService(context.Background(), &types.Tenant{ID: 8}, backend.ID, "local", t.TempDir())
require.Error(t, err)
}
+37 -3
View File
@@ -21,12 +21,13 @@ type ListTenantsParams struct {
// tenantService implements the TenantService interface
type tenantService struct {
repo interfaces.TenantRepository // Repository for tenant data operations
repo interfaces.TenantRepository // Repository for tenant data operations
storageRepo interfaces.StorageBackendRepository
}
// NewTenantService creates a new tenant service instance
func NewTenantService(repo interfaces.TenantRepository) interfaces.TenantService {
return &tenantService{repo: repo}
func NewTenantService(repo interfaces.TenantRepository, storageRepo interfaces.StorageBackendRepository) interfaces.TenantService {
return &tenantService{repo: repo, storageRepo: storageRepo}
}
// CreateTenant creates a new tenant
@@ -60,11 +61,44 @@ func (s *tenantService) CreateTenant(ctx context.Context, tenant *types.Tenant)
})
return nil, err
}
if err := s.createDefaultStorageBackend(ctx, tenant); err != nil {
// No related rows exist yet, so rolling the tenant back is safe and
// avoids leaving a workspace that cannot bind new knowledge bases.
_ = s.repo.DeleteTenant(ctx, tenant.ID)
return nil, err
}
logger.Infof(ctx, "Tenant created successfully, ID: %d, name: %s", tenant.ID, tenant.Name)
return tenant, nil
}
func (s *tenantService) createDefaultStorageBackend(ctx context.Context, tenant *types.Tenant) error {
if s.storageRepo == nil || tenant == nil {
return nil
}
provider := ""
if tenant.StorageEngineConfig != nil {
provider = tenant.StorageEngineConfig.DefaultProvider
}
backend := types.StorageBackendFromLegacy(tenant.ID, provider, tenant.StorageEngineConfig)
if backend == nil {
backend = types.StorageBackendFromEnvironment(tenant.ID)
}
if backend == nil {
return errors.New("no supported default storage backend is configured")
}
backend.LegacyAlias = true
if err := s.storageRepo.Create(ctx, backend); err != nil {
return err
}
tenant.DefaultStorageBackendID = &backend.ID
if err := s.repo.UpdateTenant(ctx, tenant); err != nil {
_ = s.storageRepo.Delete(ctx, tenant.ID, backend.ID)
return err
}
return nil
}
// GetTenantByID retrieves a tenant by their ID
func (s *tenantService) GetTenantByID(ctx context.Context, id uint64) (*types.Tenant, error) {
if id == 0 {
@@ -0,0 +1,35 @@
package service_test
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func TestCreateTenantCreatesConcreteDefaultStorageBackend(t *testing.T) {
t.Setenv("STORAGE_TYPE", "local")
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&types.Tenant{}, &types.StorageBackend{}))
tenantRepo := repository.NewTenantRepository(db)
storageRepo := repository.NewStorageBackendRepository(db)
tenantSvc := service.NewTenantService(tenantRepo, storageRepo)
tenant, err := tenantSvc.CreateTenant(context.Background(), &types.Tenant{Name: "workspace"})
require.NoError(t, err)
require.NotNil(t, tenant.DefaultStorageBackendID)
backend, err := storageRepo.GetByID(context.Background(), tenant.ID, *tenant.DefaultStorageBackendID)
require.NoError(t, err)
require.NotNil(t, backend)
assert.Equal(t, "local", backend.Provider)
assert.Equal(t, types.StorageBackendSourceEnv, backend.Source)
assert.True(t, backend.LegacyAlias)
}
@@ -137,7 +137,7 @@ func mockEngineFactory(err error) interfaces.EngineFactory {
// mockEngineService satisfies interfaces.RetrieveEngineService minimally.
type mockEngineService struct{}
func (m *mockEngineService) EngineType() types.RetrieverEngineType { return "mock" }
func (m *mockEngineService) EngineType() types.RetrieverEngineType { return "mock" }
func (m *mockEngineService) Retrieve(_ context.Context, _ types.RetrieveParams) ([]*types.RetrieveResult, error) {
return nil, nil
}
@@ -842,7 +842,8 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
is_pinned INTEGER NOT NULL DEFAULT 0,
pinned_at DATETIME NULL,
asr_config TEXT,
vector_store_id VARCHAR(36),
vector_store_id VARCHAR(36),
storage_backend_id VARCHAR(36),
wiki_config TEXT,
indexing_strategy TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
@@ -1077,7 +1078,7 @@ func TestDeleteStore_Guard_RejectsCount3(t *testing.T) {
storeID := "store-A"
for i := 0; i < 3; i++ {
kb := &types.KnowledgeBase{
ID: "kb-multi-" + tenantID2s(1) + "-" + ptrOrEmpty(&storeID) + "-" + string(rune('a'+i)),
ID: "kb-multi-" + tenantID2s(1) + "-" + ptrOrEmpty(&storeID) + "-" + string(rune('a'+i)),
Name: "kb", TenantID: 1, EmbeddingModelID: "e", SummaryModelID: "s",
VectorStoreID: &storeID,
}