feat: add volcengine tos support

This commit is contained in:
Dounx
2026-02-24 23:12:28 +08:00
committed by lyingbug
parent 172bf797ee
commit 6adecdb30e
7 changed files with 296 additions and 3 deletions
+26 -1
View File
@@ -27,7 +27,7 @@ DB_DRIVER=postgres
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant)
RETRIEVE_DRIVER=postgres
# 文件存储类型(local/minio/cos)
# 文件存储类型(local/minio/cos/tos)
STORAGE_TYPE=local
# 流处理后端(memory/redis)
@@ -169,6 +169,31 @@ CONCURRENCY_POOL_SIZE=5
# COS_ENABLE_OLD_DOMAIN=true 表示启用旧的域名格式,默认为 true
COS_ENABLE_OLD_DOMAIN=true
# 如果使用火山引擎TOS作为文件存储,需要配置以下参数
# 火山引擎TOS的访问端点,例如 https://tos-cn-beijing.volces.com
# TOS_ENDPOINT=https://tos-cn-beijing.volces.com
# 火山引擎TOS的区域,例如 cn-beijing
# TOS_REGION=cn-beijing
# 火山引擎TOS访问密钥 Access Key
# TOS_ACCESS_KEY=your_tos_access_key
# 火山引擎TOS访问密钥 Secret Key
# TOS_SECRET_KEY=your_tos_secret_key
# 火山引擎TOS桶名称
# TOS_BUCKET_NAME=your_tos_bucket_name
# 火山引擎TOS可选路径前缀(可选)
# TOS_PATH_PREFIX=your_tos_path_prefix
# 火山引擎TOS临时桶名称(可选,用于存放自动过期临时文件)
# TOS_TEMP_BUCKET_NAME=your_tos_temp_bucket_name
# 火山引擎TOS临时桶区域(可选,默认与主桶相同)
# TOS_TEMP_REGION=your_tos_temp_region
# 如果解析网络连接使用Web代理,需要配置以下参数
# WEB_PROXY=your_web_proxy
+1
View File
@@ -36,6 +36,7 @@ require (
github.com/swaggo/gin-swagger v1.6.1
github.com/swaggo/swag v1.16.6
github.com/tencentyun/cos-go-sdk-v5 v0.7.65
github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.23
github.com/yanyiwu/gojieba v1.4.5
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0
+4
View File
@@ -230,6 +230,7 @@ github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -330,6 +331,7 @@ github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsF
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY=
github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
@@ -385,6 +387,8 @@ github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vb
github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.23 h1:PvDvNKGXNNH8arQZRdu0zpwkaatXjkJQTiAmI3H8njA=
github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.23/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+1 -1
View File
@@ -90,7 +90,7 @@ app:
GIN_MODE: release
# -- Retrieval driver: postgres, elasticsearch_v7, elasticsearch_v8, qdrant
RETRIEVE_DRIVER: postgres
# -- Storage type: local, minio, cos
# -- Storage type: local, minio, cos, tos
STORAGE_TYPE: local
LOCAL_STORAGE_BASE_DIR: /data/files
AUTO_RECOVER_DIRTY: "true"
+238
View File
@@ -0,0 +1,238 @@
package file
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
)
// tosFileService implements the FileService interface for Volcengine TOS.
type tosFileService struct {
client *tos.ClientV2
pathPrefix string
bucketName string
tempBucketName string
}
// NewTosFileService creates a TOS file service.
func NewTosFileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) {
return NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, "", "")
}
// NewTosFileServiceWithTempBucket creates a TOS file service with optional temp bucket.
func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
client, err := tos.NewClientV2(
endpoint,
tos.WithRegion(region),
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
)
if err != nil {
return nil, fmt.Errorf("failed to initialize TOS client: %w", err)
}
if err := ensureTOSBucket(client, bucketName); err != nil {
return nil, err
}
if tempBucketName != "" {
if tempRegion == "" {
tempRegion = region
}
// Temporary bucket may belong to another region, so probe with a short-lived client.
tempClient, err := tos.NewClientV2(
endpoint,
tos.WithRegion(tempRegion),
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
)
if err != nil {
return nil, fmt.Errorf("failed to initialize TOS temp client: %w", err)
}
if err := ensureTOSBucket(tempClient, tempBucketName); err != nil {
return nil, err
}
}
return &tosFileService{
client: client,
pathPrefix: strings.Trim(pathPrefix, "/"),
bucketName: bucketName,
tempBucketName: tempBucketName,
}, nil
}
func ensureTOSBucket(client *tos.ClientV2, bucketName string) error {
_, err := client.HeadBucket(context.Background(), &tos.HeadBucketInput{
Bucket: bucketName,
})
if err == nil {
return nil
}
var serverErr *tos.TosServerError
if errors.As(err, &serverErr) && serverErr.StatusCode == 404 {
_, createErr := client.CreateBucketV2(context.Background(), &tos.CreateBucketV2Input{
Bucket: bucketName,
})
if createErr == nil {
return nil
}
if errors.As(createErr, &serverErr) && serverErr.StatusCode == 409 {
return nil
}
return fmt.Errorf("failed to create TOS bucket: %w", createErr)
}
return fmt.Errorf("failed to check TOS bucket: %w", err)
}
func joinTOSObjectKey(parts ...string) string {
filtered := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.Trim(part, "/")
if part != "" {
filtered = append(filtered, part)
}
}
return strings.Join(filtered, "/")
}
func parseTOSFilePath(filePath string) (bucketName string, objectKey string, err error) {
const prefix = "tos://"
if !strings.HasPrefix(filePath, prefix) {
return "", "", fmt.Errorf("invalid TOS file path: %s", filePath)
}
rest := strings.TrimPrefix(filePath, prefix)
parts := strings.SplitN(rest, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return "", "", fmt.Errorf("invalid TOS file path: %s", filePath)
}
return parts[0], parts[1], nil
}
func (s *tosFileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
ext := filepath.Ext(file.Filename)
objectName := joinTOSObjectKey(
s.pathPrefix,
fmt.Sprintf("%d", tenantID),
knowledgeID,
uuid.New().String()+ext,
)
src, err := file.Open()
if err != nil {
return "", fmt.Errorf("failed to open file: %w", err)
}
defer src.Close()
_, err = s.client.PutObjectV2(ctx, &tos.PutObjectV2Input{
PutObjectBasicInput: tos.PutObjectBasicInput{
Bucket: s.bucketName,
Key: objectName,
ContentType: file.Header.Get("Content-Type"),
},
Content: src,
})
if err != nil {
return "", fmt.Errorf("failed to upload file to TOS: %w", err)
}
return fmt.Sprintf("tos://%s/%s", s.bucketName, objectName), nil
}
func (s *tosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
ext := filepath.Ext(fileName)
reader := bytes.NewReader(data)
targetBucket := s.bucketName
objectName := joinTOSObjectKey(
s.pathPrefix,
fmt.Sprintf("%d", tenantID),
"exports",
uuid.New().String()+ext,
)
if temp && s.tempBucketName != "" {
targetBucket = s.tempBucketName
objectName = joinTOSObjectKey(
"exports",
fmt.Sprintf("%d", tenantID),
uuid.New().String()+ext,
)
}
_, err := s.client.PutObjectV2(ctx, &tos.PutObjectV2Input{
PutObjectBasicInput: tos.PutObjectBasicInput{
Bucket: targetBucket,
Key: objectName,
ContentType: "text/csv; charset=utf-8",
},
Content: reader,
})
if err != nil {
return "", fmt.Errorf("failed to upload bytes to TOS: %w", err)
}
return fmt.Sprintf("tos://%s/%s", targetBucket, objectName), nil
}
func (s *tosFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
bucketName, objectName, err := parseTOSFilePath(filePath)
if err != nil {
return nil, err
}
output, err := s.client.GetObjectV2(ctx, &tos.GetObjectV2Input{
Bucket: bucketName,
Key: objectName,
})
if err != nil {
return nil, fmt.Errorf("failed to get file from TOS: %w", err)
}
return output.Content, nil
}
func (s *tosFileService) DeleteFile(ctx context.Context, filePath string) error {
bucketName, objectName, err := parseTOSFilePath(filePath)
if err != nil {
return err
}
_, err = s.client.DeleteObjectV2(ctx, &tos.DeleteObjectV2Input{
Bucket: bucketName,
Key: objectName,
})
if err != nil {
return fmt.Errorf("failed to delete file from TOS: %w", err)
}
return nil
}
func (s *tosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
bucketName, objectName, err := parseTOSFilePath(filePath)
if err != nil {
return "", err
}
output, err := s.client.PreSignedURL(&tos.PreSignedURLInput{
HTTPMethod: enum.HttpMethodGet,
Bucket: bucketName,
Key: objectName,
Expires: int64((24 * time.Hour).Seconds()),
})
if err != nil {
return "", fmt.Errorf("failed to generate TOS presigned URL: %w", err)
}
return output.SignedUrl, nil
}
+18
View File
@@ -414,6 +414,24 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
)
case "tos":
if os.Getenv("TOS_ENDPOINT") == "" ||
os.Getenv("TOS_REGION") == "" ||
os.Getenv("TOS_ACCESS_KEY") == "" ||
os.Getenv("TOS_SECRET_KEY") == "" ||
os.Getenv("TOS_BUCKET_NAME") == "" {
return nil, fmt.Errorf("missing TOS configuration")
}
return file.NewTosFileServiceWithTempBucket(
os.Getenv("TOS_ENDPOINT"),
os.Getenv("TOS_REGION"),
os.Getenv("TOS_ACCESS_KEY"),
os.Getenv("TOS_SECRET_KEY"),
os.Getenv("TOS_BUCKET_NAME"),
os.Getenv("TOS_PATH_PREFIX"),
os.Getenv("TOS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
os.Getenv("TOS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
)
case "local":
return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil
case "dummy":
+8 -1
View File
@@ -89,6 +89,14 @@ if [ "$STORAGE_TYPE" = "minio" ]; then
check_var "MINIO_BUCKET_NAME"
fi
if [ "$STORAGE_TYPE" = "tos" ]; then
check_var "TOS_ENDPOINT"
check_var "TOS_REGION"
check_var "TOS_ACCESS_KEY"
check_var "TOS_SECRET_KEY"
check_var "TOS_BUCKET_NAME"
fi
echo ""
log_info "Redis 配置:"
check_var "REDIS_ADDR"
@@ -193,4 +201,3 @@ printf "%b\n" "${GREEN}========================================${NC}"
echo ""
exit $errors