feat: add volcengine tos support
This commit is contained in:
+26
-1
@@ -27,7 +27,7 @@ DB_DRIVER=postgres
|
||||
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant)
|
||||
RETRIEVE_DRIVER=postgres
|
||||
|
||||
# 文件存储类型(local/minio/cos)
|
||||
# 文件存储类型(local/minio/cos/tos)
|
||||
STORAGE_TYPE=local
|
||||
|
||||
# 流处理后端(memory/redis)
|
||||
@@ -169,6 +169,31 @@ CONCURRENCY_POOL_SIZE=5
|
||||
# COS_ENABLE_OLD_DOMAIN=true 表示启用旧的域名格式,默认为 true
|
||||
COS_ENABLE_OLD_DOMAIN=true
|
||||
|
||||
# 如果使用火山引擎TOS作为文件存储,需要配置以下参数
|
||||
# 火山引擎TOS的访问端点,例如 https://tos-cn-beijing.volces.com
|
||||
# TOS_ENDPOINT=https://tos-cn-beijing.volces.com
|
||||
|
||||
# 火山引擎TOS的区域,例如 cn-beijing
|
||||
# TOS_REGION=cn-beijing
|
||||
|
||||
# 火山引擎TOS访问密钥 Access Key
|
||||
# TOS_ACCESS_KEY=your_tos_access_key
|
||||
|
||||
# 火山引擎TOS访问密钥 Secret Key
|
||||
# TOS_SECRET_KEY=your_tos_secret_key
|
||||
|
||||
# 火山引擎TOS桶名称
|
||||
# TOS_BUCKET_NAME=your_tos_bucket_name
|
||||
|
||||
# 火山引擎TOS可选路径前缀(可选)
|
||||
# TOS_PATH_PREFIX=your_tos_path_prefix
|
||||
|
||||
# 火山引擎TOS临时桶名称(可选,用于存放自动过期临时文件)
|
||||
# TOS_TEMP_BUCKET_NAME=your_tos_temp_bucket_name
|
||||
|
||||
# 火山引擎TOS临时桶区域(可选,默认与主桶相同)
|
||||
# TOS_TEMP_REGION=your_tos_temp_region
|
||||
|
||||
# 如果解析网络连接使用Web代理,需要配置以下参数
|
||||
# WEB_PROXY=your_web_proxy
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ require (
|
||||
github.com/swaggo/gin-swagger v1.6.1
|
||||
github.com/swaggo/swag v1.16.6
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.65
|
||||
github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.23
|
||||
github.com/yanyiwu/gojieba v1.4.5
|
||||
go.opentelemetry.io/otel v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0
|
||||
|
||||
@@ -230,6 +230,7 @@ github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -330,6 +331,7 @@ github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsF
|
||||
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
|
||||
github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY=
|
||||
github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
@@ -385,6 +387,8 @@ github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vb
|
||||
github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.23 h1:PvDvNKGXNNH8arQZRdu0zpwkaatXjkJQTiAmI3H8njA=
|
||||
github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.23/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ app:
|
||||
GIN_MODE: release
|
||||
# -- Retrieval driver: postgres, elasticsearch_v7, elasticsearch_v8, qdrant
|
||||
RETRIEVE_DRIVER: postgres
|
||||
# -- Storage type: local, minio, cos
|
||||
# -- Storage type: local, minio, cos, tos
|
||||
STORAGE_TYPE: local
|
||||
LOCAL_STORAGE_BASE_DIR: /data/files
|
||||
AUTO_RECOVER_DIRTY: "true"
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
|
||||
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
|
||||
)
|
||||
|
||||
// tosFileService implements the FileService interface for Volcengine TOS.
|
||||
type tosFileService struct {
|
||||
client *tos.ClientV2
|
||||
pathPrefix string
|
||||
bucketName string
|
||||
tempBucketName string
|
||||
}
|
||||
|
||||
// NewTosFileService creates a TOS file service.
|
||||
func NewTosFileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) {
|
||||
return NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, "", "")
|
||||
}
|
||||
|
||||
// NewTosFileServiceWithTempBucket creates a TOS file service with optional temp bucket.
|
||||
func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
client, err := tos.NewClientV2(
|
||||
endpoint,
|
||||
tos.WithRegion(region),
|
||||
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize TOS client: %w", err)
|
||||
}
|
||||
|
||||
if err := ensureTOSBucket(client, bucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if tempBucketName != "" {
|
||||
if tempRegion == "" {
|
||||
tempRegion = region
|
||||
}
|
||||
// Temporary bucket may belong to another region, so probe with a short-lived client.
|
||||
tempClient, err := tos.NewClientV2(
|
||||
endpoint,
|
||||
tos.WithRegion(tempRegion),
|
||||
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize TOS temp client: %w", err)
|
||||
}
|
||||
if err := ensureTOSBucket(tempClient, tempBucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return &tosFileService{
|
||||
client: client,
|
||||
pathPrefix: strings.Trim(pathPrefix, "/"),
|
||||
bucketName: bucketName,
|
||||
tempBucketName: tempBucketName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ensureTOSBucket(client *tos.ClientV2, bucketName string) error {
|
||||
_, err := client.HeadBucket(context.Background(), &tos.HeadBucketInput{
|
||||
Bucket: bucketName,
|
||||
})
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var serverErr *tos.TosServerError
|
||||
if errors.As(err, &serverErr) && serverErr.StatusCode == 404 {
|
||||
_, createErr := client.CreateBucketV2(context.Background(), &tos.CreateBucketV2Input{
|
||||
Bucket: bucketName,
|
||||
})
|
||||
if createErr == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.As(createErr, &serverErr) && serverErr.StatusCode == 409 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to create TOS bucket: %w", createErr)
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to check TOS bucket: %w", err)
|
||||
}
|
||||
|
||||
func joinTOSObjectKey(parts ...string) string {
|
||||
filtered := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.Trim(part, "/")
|
||||
if part != "" {
|
||||
filtered = append(filtered, part)
|
||||
}
|
||||
}
|
||||
return strings.Join(filtered, "/")
|
||||
}
|
||||
|
||||
func parseTOSFilePath(filePath string) (bucketName string, objectKey string, err error) {
|
||||
const prefix = "tos://"
|
||||
if !strings.HasPrefix(filePath, prefix) {
|
||||
return "", "", fmt.Errorf("invalid TOS file path: %s", filePath)
|
||||
}
|
||||
|
||||
rest := strings.TrimPrefix(filePath, prefix)
|
||||
parts := strings.SplitN(rest, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", fmt.Errorf("invalid TOS file path: %s", filePath)
|
||||
}
|
||||
return parts[0], parts[1], nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
|
||||
ext := filepath.Ext(file.Filename)
|
||||
objectName := joinTOSObjectKey(
|
||||
s.pathPrefix,
|
||||
fmt.Sprintf("%d", tenantID),
|
||||
knowledgeID,
|
||||
uuid.New().String()+ext,
|
||||
)
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
_, err = s.client.PutObjectV2(ctx, &tos.PutObjectV2Input{
|
||||
PutObjectBasicInput: tos.PutObjectBasicInput{
|
||||
Bucket: s.bucketName,
|
||||
Key: objectName,
|
||||
ContentType: file.Header.Get("Content-Type"),
|
||||
},
|
||||
Content: src,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to TOS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("tos://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
ext := filepath.Ext(fileName)
|
||||
reader := bytes.NewReader(data)
|
||||
|
||||
targetBucket := s.bucketName
|
||||
objectName := joinTOSObjectKey(
|
||||
s.pathPrefix,
|
||||
fmt.Sprintf("%d", tenantID),
|
||||
"exports",
|
||||
uuid.New().String()+ext,
|
||||
)
|
||||
|
||||
if temp && s.tempBucketName != "" {
|
||||
targetBucket = s.tempBucketName
|
||||
objectName = joinTOSObjectKey(
|
||||
"exports",
|
||||
fmt.Sprintf("%d", tenantID),
|
||||
uuid.New().String()+ext,
|
||||
)
|
||||
}
|
||||
|
||||
_, err := s.client.PutObjectV2(ctx, &tos.PutObjectV2Input{
|
||||
PutObjectBasicInput: tos.PutObjectBasicInput{
|
||||
Bucket: targetBucket,
|
||||
Key: objectName,
|
||||
ContentType: "text/csv; charset=utf-8",
|
||||
},
|
||||
Content: reader,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to TOS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("tos://%s/%s", targetBucket, objectName), nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
bucketName, objectName, err := parseTOSFilePath(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
output, err := s.client.GetObjectV2(ctx, &tos.GetObjectV2Input{
|
||||
Bucket: bucketName,
|
||||
Key: objectName,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file from TOS: %w", err)
|
||||
}
|
||||
return output.Content, nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
bucketName, objectName, err := parseTOSFilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = s.client.DeleteObjectV2(ctx, &tos.DeleteObjectV2Input{
|
||||
Bucket: bucketName,
|
||||
Key: objectName,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file from TOS: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
bucketName, objectName, err := parseTOSFilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
output, err := s.client.PreSignedURL(&tos.PreSignedURLInput{
|
||||
HTTPMethod: enum.HttpMethodGet,
|
||||
Bucket: bucketName,
|
||||
Key: objectName,
|
||||
Expires: int64((24 * time.Hour).Seconds()),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate TOS presigned URL: %w", err)
|
||||
}
|
||||
return output.SignedUrl, nil
|
||||
}
|
||||
@@ -414,6 +414,24 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
|
||||
os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
|
||||
)
|
||||
case "tos":
|
||||
if os.Getenv("TOS_ENDPOINT") == "" ||
|
||||
os.Getenv("TOS_REGION") == "" ||
|
||||
os.Getenv("TOS_ACCESS_KEY") == "" ||
|
||||
os.Getenv("TOS_SECRET_KEY") == "" ||
|
||||
os.Getenv("TOS_BUCKET_NAME") == "" {
|
||||
return nil, fmt.Errorf("missing TOS configuration")
|
||||
}
|
||||
return file.NewTosFileServiceWithTempBucket(
|
||||
os.Getenv("TOS_ENDPOINT"),
|
||||
os.Getenv("TOS_REGION"),
|
||||
os.Getenv("TOS_ACCESS_KEY"),
|
||||
os.Getenv("TOS_SECRET_KEY"),
|
||||
os.Getenv("TOS_BUCKET_NAME"),
|
||||
os.Getenv("TOS_PATH_PREFIX"),
|
||||
os.Getenv("TOS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
|
||||
os.Getenv("TOS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
|
||||
)
|
||||
case "local":
|
||||
return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil
|
||||
case "dummy":
|
||||
|
||||
@@ -89,6 +89,14 @@ if [ "$STORAGE_TYPE" = "minio" ]; then
|
||||
check_var "MINIO_BUCKET_NAME"
|
||||
fi
|
||||
|
||||
if [ "$STORAGE_TYPE" = "tos" ]; then
|
||||
check_var "TOS_ENDPOINT"
|
||||
check_var "TOS_REGION"
|
||||
check_var "TOS_ACCESS_KEY"
|
||||
check_var "TOS_SECRET_KEY"
|
||||
check_var "TOS_BUCKET_NAME"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_info "Redis 配置:"
|
||||
check_var "REDIS_ADDR"
|
||||
@@ -193,4 +201,3 @@ printf "%b\n" "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
exit $errors
|
||||
|
||||
|
||||
Reference in New Issue
Block a user