Files
knowledge-governance/internal/releaseadapter/v071/s3_object_store.go
T
姚章浩 168fc2ecb7 fix(release): 对象存储四个入口共用同一定位符解析
真实资源路径是 storage://<backend>/<provider>://<bucket>/<key>。Open/Put 用
validObjectKey 并拆出裸 key,而 Stat/Delete/Copy 用只认单层的 validObjectLocator
且把整串当对象键——对象在桶里却校验读不到。集成用例只喂裸 key,把这条不一致
藏了过去;写集合用例里那句"读不到也算通过"同样把断言关掉了。

现在四个入口共用 validObjectKey + providerPathFromLocator;集成用例喂真实定位符,
负例覆盖键不存在 / provider 不符 / 路径穿越 / 复制源不存在,并验证 Delete 后再读必失败。
2026-10-30 01:33:45 +08:00

179 lines
7.2 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type S3ObjectStoreConfig struct {
Provider string
Endpoint string
Region string
AccessKeyID string
SecretAccessKey string
Bucket string
Secure bool
ForcePathStyle bool
MaxObjectBytes int64
HTTPClient *http.Client
}
type S3ObjectStore struct {
provider string
bucket string
maxObjectBytes int64
client *s3.Client
}
func NewS3ObjectStore(config S3ObjectStoreConfig) (*S3ObjectStore, error) {
config.Provider = strings.TrimSpace(config.Provider)
config.Endpoint = strings.TrimSpace(config.Endpoint)
config.Region = strings.TrimSpace(config.Region)
config.Bucket = strings.TrimSpace(config.Bucket)
if config.Region == "" {
config.Region = "us-east-1"
}
if config.MaxObjectBytes == 0 {
config.MaxObjectBytes = 100 * 1024 * 1024
}
endpoint, err := url.Parse(config.Endpoint)
if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.Path != "" || endpoint.RawQuery != "" || endpoint.Fragment != "" ||
(endpoint.Scheme != "http" && endpoint.Scheme != "https") || (config.Secure && endpoint.Scheme != "https") ||
!validObjectProvider(config.Provider) || config.AccessKeyID == "" || config.SecretAccessKey == "" || config.Bucket == "" || config.MaxObjectBytes < 1 {
return nil, errors.New("invalid S3 object store configuration")
}
httpClient := &http.Client{Timeout: 2 * time.Minute}
if config.HTTPClient != nil {
clone := *config.HTTPClient
httpClient = &clone
}
httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("S3 redirect rejected") }
forcePathStyle := config.ForcePathStyle || config.Provider == "minio"
client := s3.New(s3.Options{
Region: config.Region,
BaseEndpoint: aws.String(endpoint.String()),
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.SecretAccessKey, "")),
UsePathStyle: forcePathStyle,
HTTPClient: httpClient,
})
return &S3ObjectStore{provider: config.Provider, bucket: config.Bucket, maxObjectBytes: config.MaxObjectBytes, client: client}, nil
}
func (store *S3ObjectStore) EnsureBucket(ctx context.Context) error {
if _, err := store.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(store.bucket)}); err == nil {
return nil
}
if _, err := store.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(store.bucket)}); err != nil {
return errors.New("ensure release object bucket")
}
return nil
}
func (store *S3ObjectStore) Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) {
if provider != store.provider || !validObjectKey(physicalPath) {
return nil, errors.New("invalid release object source")
}
objectPath, ok := providerPathFromLocator(physicalPath)
if !ok {
return nil, errors.New("invalid release object source")
}
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil {
return nil, errors.New("read release source object")
}
return object.Body, nil
}
func (store *S3ObjectStore) Put(ctx context.Context, provider, objectPath string, reader io.Reader, size int64) error {
if provider != store.provider || !validObjectKey(objectPath) || size < 0 || size > store.maxObjectBytes {
return errors.New("invalid release object write")
}
// 真实资源路径是 storage://<backend>/oss://<bucket>/<key>:
// 落盘只写配置的桶,所以必须先把定位符拆成裸 key,不能整串当 key。
key, ok := providerPathFromLocator(objectPath)
if !ok {
return errors.New("invalid release object write")
}
if _, err := store.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(key), Body: reader, ContentLength: aws.Int64(size)}); err != nil {
return fmt.Errorf("write release object: %w", err)
}
return nil
}
// Delete 删除单个对象;只用于显式清理(保留策略未接线,不会自动调用)。
func (store *S3ObjectStore) Delete(ctx context.Context, provider, objectPath string) error {
// 与 Open/Put 同一个校验:真实资源路径是 storage://<backend>/<provider>://<bucket>/<key>,
// 只有 validObjectKey 接受这种嵌套形状(validObjectLocator 只认单层)。
if provider != store.provider || !validObjectKey(objectPath) {
return errors.New("invalid release object delete")
}
key, ok := providerPathFromLocator(objectPath)
if !ok {
return errors.New("invalid release object delete")
}
if _, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(key)}); err != nil {
return errors.New("delete release object")
}
return nil
}
// Stat 的入参形状必须与 Open/Put 完全相同:真实资源路径是
// storage://<backend>/<provider>://<bucket>/<key>,而落盘/读取都只认裸 key。
// 曾经这里把整串当 key,于是 Put/Open 能成功、Stat 必然 404——"对象在桶里但
// 校验说读不到"。测试也只喂裸 key,绕开了这条不一致。
func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath string) (int64, string, error) {
if provider != store.provider || !validObjectKey(objectPath) {
return 0, "", errors.New("invalid release object read")
}
key, ok := providerPathFromLocator(objectPath)
if !ok {
return 0, "", errors.New("invalid release object read")
}
head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(key)})
if err != nil || head.ContentLength == nil || *head.ContentLength < 0 || *head.ContentLength > store.maxObjectBytes {
return 0, "", errors.New("inspect release object")
}
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(key)})
if err != nil {
return 0, "", errors.New("read release object")
}
defer object.Body.Close()
hash := sha256.New()
written, err := io.Copy(hash, io.LimitReader(object.Body, store.maxObjectBytes+1))
if err != nil || written != *head.ContentLength || written > store.maxObjectBytes {
return 0, "", errors.New("verify release object")
}
return written, hex.EncodeToString(hash.Sum(nil)), nil
}
func (store *S3ObjectStore) Copy(ctx context.Context, provider, sourcePath, targetPath string) error {
if provider != store.provider || !validObjectKey(sourcePath) || !validObjectKey(targetPath) {
return errors.New("invalid release object copy")
}
sourceKey, sourceOK := providerPathFromLocator(sourcePath)
targetKey, targetOK := providerPathFromLocator(targetPath)
if !sourceOK || !targetOK {
return errors.New("invalid release object copy")
}
copySource := url.PathEscape(store.bucket + "/" + sourceKey)
if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetKey), CopySource: aws.String(copySource)}); err != nil {
return errors.New("copy release object")
}
return nil
}
var _ ObjectStore = (*S3ObjectStore)(nil)
var _ ObjectReader = (*S3ObjectStore)(nil)