diff --git a/.env.example b/.env.example index 25c56c4d..39bdafff 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,7 @@ DB_DRIVER=postgres # 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant) RETRIEVE_DRIVER=postgres -# 文件存储类型(local/minio/cos) +# 文件存储类型(local/minio/cos/tos) STORAGE_TYPE=local # 流处理后端(memory/redis) @@ -169,6 +169,31 @@ CONCURRENCY_POOL_SIZE=5 # COS_ENABLE_OLD_DOMAIN=true 表示启用旧的域名格式,默认为 true COS_ENABLE_OLD_DOMAIN=true +# 如果使用火山引擎TOS作为文件存储,需要配置以下参数 +# 火山引擎TOS的访问端点,例如 https://tos-cn-beijing.volces.com +# TOS_ENDPOINT=https://tos-cn-beijing.volces.com + +# 火山引擎TOS的区域,例如 cn-beijing +# TOS_REGION=cn-beijing + +# 火山引擎TOS访问密钥 Access Key +# TOS_ACCESS_KEY=your_tos_access_key + +# 火山引擎TOS访问密钥 Secret Key +# TOS_SECRET_KEY=your_tos_secret_key + +# 火山引擎TOS桶名称 +# TOS_BUCKET_NAME=your_tos_bucket_name + +# 火山引擎TOS可选路径前缀(可选) +# TOS_PATH_PREFIX=your_tos_path_prefix + +# 火山引擎TOS临时桶名称(可选,用于存放自动过期临时文件) +# TOS_TEMP_BUCKET_NAME=your_tos_temp_bucket_name + +# 火山引擎TOS临时桶区域(可选,默认与主桶相同) +# TOS_TEMP_REGION=your_tos_temp_region + # 如果解析网络连接使用Web代理,需要配置以下参数 # WEB_PROXY=your_web_proxy diff --git a/go.mod b/go.mod index 09e00f00..6cf8f3b1 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/swaggo/gin-swagger v1.6.1 github.com/swaggo/swag v1.16.6 github.com/tencentyun/cos-go-sdk-v5 v0.7.65 + github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.23 github.com/yanyiwu/gojieba v1.4.5 go.opentelemetry.io/otel v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 diff --git a/go.sum b/go.sum index 11706d90..b46637b7 100644 --- a/go.sum +++ b/go.sum @@ -230,6 +230,7 @@ github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -330,6 +331,7 @@ github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsF github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -385,6 +387,8 @@ github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vb github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.23 h1:PvDvNKGXNNH8arQZRdu0zpwkaatXjkJQTiAmI3H8njA= +github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.23/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/helm/values.yaml b/helm/values.yaml index c928c217..f23df9a6 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -90,7 +90,7 @@ app: GIN_MODE: release # -- Retrieval driver: postgres, elasticsearch_v7, elasticsearch_v8, qdrant RETRIEVE_DRIVER: postgres - # -- Storage type: local, minio, cos + # -- Storage type: local, minio, cos, tos STORAGE_TYPE: local LOCAL_STORAGE_BASE_DIR: /data/files AUTO_RECOVER_DIRTY: "true" diff --git a/internal/application/service/file/tos.go b/internal/application/service/file/tos.go new file mode 100644 index 00000000..31d1ddfd --- /dev/null +++ b/internal/application/service/file/tos.go @@ -0,0 +1,238 @@ +package file + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "mime/multipart" + "path/filepath" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" + "github.com/volcengine/ve-tos-golang-sdk/v2/tos" + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" +) + +// tosFileService implements the FileService interface for Volcengine TOS. +type tosFileService struct { + client *tos.ClientV2 + pathPrefix string + bucketName string + tempBucketName string +} + +// NewTosFileService creates a TOS file service. +func NewTosFileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) { + return NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, "", "") +} + +// NewTosFileServiceWithTempBucket creates a TOS file service with optional temp bucket. +func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) { + client, err := tos.NewClientV2( + endpoint, + tos.WithRegion(region), + tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)), + ) + if err != nil { + return nil, fmt.Errorf("failed to initialize TOS client: %w", err) + } + + if err := ensureTOSBucket(client, bucketName); err != nil { + return nil, err + } + + if tempBucketName != "" { + if tempRegion == "" { + tempRegion = region + } + // Temporary bucket may belong to another region, so probe with a short-lived client. + tempClient, err := tos.NewClientV2( + endpoint, + tos.WithRegion(tempRegion), + tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)), + ) + if err != nil { + return nil, fmt.Errorf("failed to initialize TOS temp client: %w", err) + } + if err := ensureTOSBucket(tempClient, tempBucketName); err != nil { + return nil, err + } + } + + return &tosFileService{ + client: client, + pathPrefix: strings.Trim(pathPrefix, "/"), + bucketName: bucketName, + tempBucketName: tempBucketName, + }, nil +} + +func ensureTOSBucket(client *tos.ClientV2, bucketName string) error { + _, err := client.HeadBucket(context.Background(), &tos.HeadBucketInput{ + Bucket: bucketName, + }) + if err == nil { + return nil + } + + var serverErr *tos.TosServerError + if errors.As(err, &serverErr) && serverErr.StatusCode == 404 { + _, createErr := client.CreateBucketV2(context.Background(), &tos.CreateBucketV2Input{ + Bucket: bucketName, + }) + if createErr == nil { + return nil + } + if errors.As(createErr, &serverErr) && serverErr.StatusCode == 409 { + return nil + } + return fmt.Errorf("failed to create TOS bucket: %w", createErr) + } + + return fmt.Errorf("failed to check TOS bucket: %w", err) +} + +func joinTOSObjectKey(parts ...string) string { + filtered := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.Trim(part, "/") + if part != "" { + filtered = append(filtered, part) + } + } + return strings.Join(filtered, "/") +} + +func parseTOSFilePath(filePath string) (bucketName string, objectKey string, err error) { + const prefix = "tos://" + if !strings.HasPrefix(filePath, prefix) { + return "", "", fmt.Errorf("invalid TOS file path: %s", filePath) + } + + rest := strings.TrimPrefix(filePath, prefix) + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid TOS file path: %s", filePath) + } + return parts[0], parts[1], nil +} + +func (s *tosFileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) { + ext := filepath.Ext(file.Filename) + objectName := joinTOSObjectKey( + s.pathPrefix, + fmt.Sprintf("%d", tenantID), + knowledgeID, + uuid.New().String()+ext, + ) + + src, err := file.Open() + if err != nil { + return "", fmt.Errorf("failed to open file: %w", err) + } + defer src.Close() + + _, err = s.client.PutObjectV2(ctx, &tos.PutObjectV2Input{ + PutObjectBasicInput: tos.PutObjectBasicInput{ + Bucket: s.bucketName, + Key: objectName, + ContentType: file.Header.Get("Content-Type"), + }, + Content: src, + }) + if err != nil { + return "", fmt.Errorf("failed to upload file to TOS: %w", err) + } + + return fmt.Sprintf("tos://%s/%s", s.bucketName, objectName), nil +} + +func (s *tosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) { + ext := filepath.Ext(fileName) + reader := bytes.NewReader(data) + + targetBucket := s.bucketName + objectName := joinTOSObjectKey( + s.pathPrefix, + fmt.Sprintf("%d", tenantID), + "exports", + uuid.New().String()+ext, + ) + + if temp && s.tempBucketName != "" { + targetBucket = s.tempBucketName + objectName = joinTOSObjectKey( + "exports", + fmt.Sprintf("%d", tenantID), + uuid.New().String()+ext, + ) + } + + _, err := s.client.PutObjectV2(ctx, &tos.PutObjectV2Input{ + PutObjectBasicInput: tos.PutObjectBasicInput{ + Bucket: targetBucket, + Key: objectName, + ContentType: "text/csv; charset=utf-8", + }, + Content: reader, + }) + if err != nil { + return "", fmt.Errorf("failed to upload bytes to TOS: %w", err) + } + + return fmt.Sprintf("tos://%s/%s", targetBucket, objectName), nil +} + +func (s *tosFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { + bucketName, objectName, err := parseTOSFilePath(filePath) + if err != nil { + return nil, err + } + + output, err := s.client.GetObjectV2(ctx, &tos.GetObjectV2Input{ + Bucket: bucketName, + Key: objectName, + }) + if err != nil { + return nil, fmt.Errorf("failed to get file from TOS: %w", err) + } + return output.Content, nil +} + +func (s *tosFileService) DeleteFile(ctx context.Context, filePath string) error { + bucketName, objectName, err := parseTOSFilePath(filePath) + if err != nil { + return err + } + + _, err = s.client.DeleteObjectV2(ctx, &tos.DeleteObjectV2Input{ + Bucket: bucketName, + Key: objectName, + }) + if err != nil { + return fmt.Errorf("failed to delete file from TOS: %w", err) + } + return nil +} + +func (s *tosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + bucketName, objectName, err := parseTOSFilePath(filePath) + if err != nil { + return "", err + } + + output, err := s.client.PreSignedURL(&tos.PreSignedURLInput{ + HTTPMethod: enum.HttpMethodGet, + Bucket: bucketName, + Key: objectName, + Expires: int64((24 * time.Hour).Seconds()), + }) + if err != nil { + return "", fmt.Errorf("failed to generate TOS presigned URL: %w", err) + } + return output.SignedUrl, nil +} diff --git a/internal/container/container.go b/internal/container/container.go index cf3ad749..c1d7c4d0 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -414,6 +414,24 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) { os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期) os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同 ) + case "tos": + if os.Getenv("TOS_ENDPOINT") == "" || + os.Getenv("TOS_REGION") == "" || + os.Getenv("TOS_ACCESS_KEY") == "" || + os.Getenv("TOS_SECRET_KEY") == "" || + os.Getenv("TOS_BUCKET_NAME") == "" { + return nil, fmt.Errorf("missing TOS configuration") + } + return file.NewTosFileServiceWithTempBucket( + os.Getenv("TOS_ENDPOINT"), + os.Getenv("TOS_REGION"), + os.Getenv("TOS_ACCESS_KEY"), + os.Getenv("TOS_SECRET_KEY"), + os.Getenv("TOS_BUCKET_NAME"), + os.Getenv("TOS_PATH_PREFIX"), + os.Getenv("TOS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期) + os.Getenv("TOS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同 + ) case "local": return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil case "dummy": diff --git a/scripts/check-env.sh b/scripts/check-env.sh index fe2449ec..592f1687 100755 --- a/scripts/check-env.sh +++ b/scripts/check-env.sh @@ -89,6 +89,14 @@ if [ "$STORAGE_TYPE" = "minio" ]; then check_var "MINIO_BUCKET_NAME" fi +if [ "$STORAGE_TYPE" = "tos" ]; then + check_var "TOS_ENDPOINT" + check_var "TOS_REGION" + check_var "TOS_ACCESS_KEY" + check_var "TOS_SECRET_KEY" + check_var "TOS_BUCKET_NAME" +fi + echo "" log_info "Redis 配置:" check_var "REDIS_ADDR" @@ -193,4 +201,3 @@ printf "%b\n" "${GREEN}========================================${NC}" echo "" exit $errors -