fix(release): 对象存储四个入口共用同一定位符解析
真实资源路径是 storage://<backend>/<provider>://<bucket>/<key>。Open/Put 用 validObjectKey 并拆出裸 key,而 Stat/Delete/Copy 用只认单层的 validObjectLocator 且把整串当对象键——对象在桶里却校验读不到。集成用例只喂裸 key,把这条不一致 藏了过去;写集合用例里那句"读不到也算通过"同样把断言关掉了。 现在四个入口共用 validObjectKey + providerPathFromLocator;集成用例喂真实定位符, 负例覆盖键不存在 / provider 不符 / 路径穿越 / 复制源不存在,并验证 Delete 后再读必失败。
This commit is contained in:
@@ -622,8 +622,13 @@ func TestLibraryWikiStageHalfWriteRollbackIsolated(t *testing.T) {
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("wiki 阶段半写回滚后目标数据被改动:%s", diff)
|
||||
}
|
||||
if _, _, err := fx.objects.s3.Stat(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src")); err == nil {
|
||||
t.Log("note: 对象字节保留在桶里(恢复只回滚行,不删对象;对象清理不属于本次范围)")
|
||||
// 回滚只回滚行,不删对象:这个定位符必须仍然**读得到**,而且 Stat 要吃真实
|
||||
// 定位符(storage://<backend>/<provider>://<bucket>/<key>),不是裸 key——
|
||||
// 曾经这里写成"读不到也当通过",把 Stat 与 Open/Put 的定位符不一致盖住了。
|
||||
if size, _, err := fx.objects.s3.Stat(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src")); err != nil {
|
||||
t.Fatalf("回滚后对象字节必须仍在桶里(只回滚行):%v", err)
|
||||
} else if size <= 0 {
|
||||
t.Fatalf("回滚后对象字节大小异常:%d", size)
|
||||
}
|
||||
t.Logf("wiki half-write rolled back: %s", outcome.Rollback)
|
||||
}
|
||||
|
||||
@@ -114,24 +114,38 @@ func (store *S3ObjectStore) Put(ctx context.Context, provider, objectPath string
|
||||
|
||||
// Delete 删除单个对象;只用于显式清理(保留策略未接线,不会自动调用)。
|
||||
func (store *S3ObjectStore) Delete(ctx context.Context, provider, objectPath string) error {
|
||||
if provider != store.provider || !validObjectLocator(objectPath) {
|
||||
// 与 Open/Put 同一个校验:真实资源路径是 storage://<backend>/<provider>://<bucket>/<key>,
|
||||
// 只有 validObjectKey 接受这种嵌套形状(validObjectLocator 只认单层)。
|
||||
if provider != store.provider || !validObjectKey(objectPath) {
|
||||
return errors.New("invalid release object delete")
|
||||
}
|
||||
if _, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)}); err != nil {
|
||||
key, ok := providerPathFromLocator(objectPath)
|
||||
if !ok {
|
||||
return errors.New("invalid release object delete")
|
||||
}
|
||||
if _, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(key)}); err != nil {
|
||||
return errors.New("delete release object")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat 的入参形状必须与 Open/Put 完全相同:真实资源路径是
|
||||
// storage://<backend>/<provider>://<bucket>/<key>,而落盘/读取都只认裸 key。
|
||||
// 曾经这里把整串当 key,于是 Put/Open 能成功、Stat 必然 404——"对象在桶里但
|
||||
// 校验说读不到"。测试也只喂裸 key,绕开了这条不一致。
|
||||
func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath string) (int64, string, error) {
|
||||
if provider != store.provider || !validObjectLocator(objectPath) {
|
||||
if provider != store.provider || !validObjectKey(objectPath) {
|
||||
return 0, "", errors.New("invalid release object read")
|
||||
}
|
||||
head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
||||
key, ok := providerPathFromLocator(objectPath)
|
||||
if !ok {
|
||||
return 0, "", errors.New("invalid release object read")
|
||||
}
|
||||
head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(key)})
|
||||
if err != nil || head.ContentLength == nil || *head.ContentLength < 0 || *head.ContentLength > store.maxObjectBytes {
|
||||
return 0, "", errors.New("inspect release object")
|
||||
}
|
||||
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
||||
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(key)})
|
||||
if err != nil {
|
||||
return 0, "", errors.New("read release object")
|
||||
}
|
||||
@@ -145,11 +159,16 @@ func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath strin
|
||||
}
|
||||
|
||||
func (store *S3ObjectStore) Copy(ctx context.Context, provider, sourcePath, targetPath string) error {
|
||||
if provider != store.provider || !validObjectLocator(sourcePath) || !validObjectLocator(targetPath) {
|
||||
if provider != store.provider || !validObjectKey(sourcePath) || !validObjectKey(targetPath) {
|
||||
return errors.New("invalid release object copy")
|
||||
}
|
||||
copySource := url.PathEscape(store.bucket + "/" + sourcePath)
|
||||
if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetPath), CopySource: aws.String(copySource)}); err != nil {
|
||||
sourceKey, sourceOK := providerPathFromLocator(sourcePath)
|
||||
targetKey, targetOK := providerPathFromLocator(targetPath)
|
||||
if !sourceOK || !targetOK {
|
||||
return errors.New("invalid release object copy")
|
||||
}
|
||||
copySource := url.PathEscape(store.bucket + "/" + sourceKey)
|
||||
if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetKey), CopySource: aws.String(copySource)}); err != nil {
|
||||
return errors.New("copy release object")
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -3,11 +3,20 @@ package v071
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 这条用例必须喂**真实定位符**:WeKnora 的资源路径是
|
||||
// storage://<backend>/<provider>://<bucket>/<key>,而 Stat/Open/Put/Copy
|
||||
// 都必须把它拆成同一个裸 key。
|
||||
//
|
||||
// 历史缺陷:Stat/Delete/Copy 曾把整串定位符当对象键,而测试只喂裸 key,
|
||||
// 于是"在桶里但校验读不到"这条不一致一直没被发现。
|
||||
func TestS3ObjectStoreIntegration(t *testing.T) {
|
||||
endpoint := os.Getenv("TEST_MINIO_ENDPOINT")
|
||||
access := os.Getenv("TEST_MINIO_ACCESS_KEY")
|
||||
@@ -24,34 +33,81 @@ func TestS3ObjectStoreIntegration(t *testing.T) {
|
||||
if err := store.EnsureBucket(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 每次运行用独立前缀:同一桶里留下的旧对象不能让"读不到"的负例假通过。
|
||||
prefix := fmt.Sprintf("fix-stat-locator/%d", time.Now().UnixNano())
|
||||
key := prefix + "/source.txt"
|
||||
locator := "storage://source-backend/minio://" + bucket + "/" + key
|
||||
data := []byte("synthetic object release content")
|
||||
if err := store.Put(ctx, "minio", "stage/source.txt", bytes.NewReader(data), int64(len(data))); err != nil {
|
||||
t.Fatal(err)
|
||||
if err := store.Put(ctx, "minio", locator, bytes.NewReader(data), int64(len(data))); err != nil {
|
||||
t.Fatalf("put by locator: %v", err)
|
||||
}
|
||||
reader, err := store.Open(ctx, "minio", "storage://source-backend/stage/source.txt")
|
||||
|
||||
// 正例 1:带 storage:// 前缀的定位符 —— 与 Put 用的是同一份字节。
|
||||
size, digest, err := store.Stat(ctx, "minio", locator)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
t.Fatalf("stat by storage locator: %v", err)
|
||||
}
|
||||
opened, err := io.ReadAll(reader)
|
||||
if size != int64(len(data)) || digest != shaBytes(data) {
|
||||
t.Fatalf("stat size=%d digest=%s", size, digest)
|
||||
}
|
||||
|
||||
// 正例 2:只带 provider scheme 的定位符也算同一个对象。
|
||||
if size, digest, err = store.Stat(ctx, "minio", "minio://"+bucket+"/"+key); err != nil {
|
||||
t.Fatalf("stat by provider locator: %v", err)
|
||||
}
|
||||
if size != int64(len(data)) || digest != shaBytes(data) {
|
||||
t.Fatalf("stat by provider locator size=%d digest=%s", size, digest)
|
||||
}
|
||||
|
||||
reader, err := store.Open(ctx, "minio", locator)
|
||||
if err != nil {
|
||||
t.Fatalf("open by locator: %v", err)
|
||||
}
|
||||
opened, readErr := io.ReadAll(reader)
|
||||
closeErr := reader.Close()
|
||||
if err != nil || closeErr != nil || !bytes.Equal(opened, data) {
|
||||
t.Fatalf("opened=%q err=%v close=%v", opened, err, closeErr)
|
||||
if readErr != nil || closeErr != nil || !bytes.Equal(opened, data) {
|
||||
t.Fatalf("opened=%q err=%v close=%v", opened, readErr, closeErr)
|
||||
}
|
||||
size, digest, err := store.Stat(ctx, "minio", "stage/source.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
finalLocator := "storage://source-backend/minio://" + bucket + "/" + prefix + "/final/source.txt"
|
||||
if err := store.Copy(ctx, "minio", locator, finalLocator); err != nil {
|
||||
t.Fatalf("copy by locator: %v", err)
|
||||
}
|
||||
if size != int64(len(data)) || digest != shaBytes(data) {
|
||||
t.Fatalf("size=%d digest=%s", size, digest)
|
||||
}
|
||||
if err := store.Copy(ctx, "minio", "stage/source.txt", "final/source.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
size, digest, err = store.Stat(ctx, "minio", "final/source.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size != int64(len(data)) || digest != shaBytes(data) {
|
||||
if size, digest, err = store.Stat(ctx, "minio", finalLocator); err != nil {
|
||||
t.Fatalf("stat copied object: %v", err)
|
||||
} else if size != int64(len(data)) || digest != shaBytes(data) {
|
||||
t.Fatalf("copy size=%d digest=%s", size, digest)
|
||||
}
|
||||
|
||||
// 负例 1:桶里没有这个键 ⇒ 必须报错,不能返回 0 字节当成"对象为空"。
|
||||
if _, _, err := store.Stat(ctx, "minio", "storage://source-backend/minio://"+bucket+"/"+prefix+"/missing.txt"); err == nil {
|
||||
t.Fatal("stat of a missing key must fail")
|
||||
}
|
||||
// 负例 2:provider 不匹配 ⇒ 直接拒绝(不能拿别人家的定位符读本桶)。
|
||||
if _, _, err := store.Stat(ctx, "oss", locator); err == nil {
|
||||
t.Fatal("stat must reject a foreign provider")
|
||||
}
|
||||
// 负例 3:结构非法的定位符(穿越路径)⇒ 拒绝。
|
||||
if _, _, err := store.Stat(ctx, "minio", "storage://source-backend/minio://"+bucket+"/../"+key); err == nil {
|
||||
t.Fatal("stat must reject a traversal locator")
|
||||
}
|
||||
// 负例 4:Copy 源不存在 ⇒ 报错,不能"成功"出一个空对象。
|
||||
if err := store.Copy(ctx, "minio", "storage://source-backend/minio://"+bucket+"/"+prefix+"/missing.txt", finalLocator+".copy"); err == nil {
|
||||
t.Fatal("copy from a missing source must fail")
|
||||
}
|
||||
// 负例 5:读回来的摘要必须与写入内容不同时不相等(防止把长度当摘要)。
|
||||
if strings.TrimSpace(digest) == "" {
|
||||
t.Fatal("stat must return a content digest")
|
||||
}
|
||||
|
||||
// Delete 也必须吃同一个定位符(它和 Stat 是同一类"按路径操作"的入口)。
|
||||
if err := store.Delete(ctx, "minio", finalLocator); err != nil {
|
||||
t.Fatalf("delete by locator: %v", err)
|
||||
}
|
||||
if _, _, err := store.Stat(ctx, "minio", finalLocator); err == nil {
|
||||
t.Fatal("stat after delete must fail")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = store.Delete(context.Background(), "minio", "storage://source-backend/minio://"+bucket+"/"+key)
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user