同步把资源字节写进目标桶,但 resources.physical_path 是从快照原样搬的, 里面还写着来源桶(storage://…/minio://kgmaint-1028a/…)。目标实例之后按这一行 取图/附件就会指向测试环境的桶,或直接取不到——跨环境引用,必须在本轮修掉: 写入前换桶、键不变,定位符解析不出来就报错(不写指向别处的行)。 试过:make verify 15 项全绿;新增定位符用例(嵌套/单层/裸键/非法); 真实同步后目标桶里对象键与行内定位符一致。
This commit is contained in:
@@ -31,8 +31,15 @@
|
||||
字段等第一次写入才动态生成;首次同步因此在读取目标状态时就失败。现在"没有范围字段
|
||||
+ 确实 0 篇文档"按空基线处理,有文档却看不出归属仍然报错。
|
||||
|
||||
- **资源定位符的桶没有跟着换**:同步把字节写进目标桶,但 `resources.physical_path`
|
||||
是从快照原样搬的,里面仍写着来源桶(`storage://…/minio://kgmaint-1028a/…`)。
|
||||
目标实例之后取图/附件会指向测试环境的桶,或直接取不到。现在写入前把定位符里的桶
|
||||
换成本次目标桶(键不变),解析不出来就报错而不是写一行指向别处的资源。
|
||||
|
||||
### Tests
|
||||
|
||||
- `internal/releaseadapter/v071/resource_locator_test.go`:嵌套/单层两种定位符换桶、裸键原样、
|
||||
非法形状拒绝。
|
||||
- `internal/releaseadapter/v071/search_empty_index_test.go`:空索引=空基线、有文档无范围字段必须报错。
|
||||
- `internal/httpapi/library_capture_test.go`:队列作业 id 必须放得进 `varchar(36)` 且形状固定。
|
||||
- `internal/releaseadapter/v071/search_tenant_filter_test.go`:接受未写租户、排除他租户、
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
// 列出并读回目标桶里的对象,打印 键/字节/sha256:资源字节是否真在目标存储里,只能这样核。
|
||||
func main() {
|
||||
endpoint, bucket := os.Getenv("S3_ENDPOINT"), os.Getenv("S3_BUCKET")
|
||||
client := s3.New(s3.Options{
|
||||
Region: "us-east-1",
|
||||
BaseEndpoint: aws.String(endpoint),
|
||||
Credentials: credentials.NewStaticCredentialsProvider(os.Getenv("S3_KEY"), os.Getenv("S3_SECRET"), ""),
|
||||
UsePathStyle: true,
|
||||
})
|
||||
ctx := context.Background()
|
||||
pages := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{Bucket: aws.String(bucket)})
|
||||
for pages.HasMorePages() {
|
||||
page, err := pages.NextPage(ctx)
|
||||
if err != nil {
|
||||
fmt.Println("list error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, item := range page.Contents {
|
||||
out, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: item.Key})
|
||||
if err != nil {
|
||||
fmt.Println("get error:", item.Key, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
hash := sha256.New()
|
||||
size, _ := io.Copy(hash, out.Body)
|
||||
out.Body.Close()
|
||||
fmt.Printf("%s\t%d\t%s\n", *item.Key, size, hex.EncodeToString(hash.Sum(nil)))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -54,8 +54,9 @@ func buildLibraryProductionStack(ctx context.Context, config appconfig.Config, s
|
||||
target, err := v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{
|
||||
Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL,
|
||||
SearchIndex: config.LibraryProductionIndex, Objects: objects, ObjectsReader: objects,
|
||||
ArchiveRoot: config.LibraryArchiveRoot,
|
||||
MaxDocuments: 200000,
|
||||
ArchiveRoot: config.LibraryArchiveRoot,
|
||||
ObjectsBucket: config.LibraryProductionS3Bucket,
|
||||
MaxDocuments: 200000,
|
||||
}, identity)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
||||
@@ -47,6 +47,9 @@ type ProductionLibraryConfig struct {
|
||||
Objects ObjectStore
|
||||
ObjectsReader ObjectReader
|
||||
ArchiveRoot string
|
||||
// ObjectsBucket 是目标对象存储的桶名:资源行里的定位符要换成它,
|
||||
// 否则同步过去的图/附件仍指向来源桶。
|
||||
ObjectsBucket string
|
||||
MaxDocuments int
|
||||
HTTPClient *http.Client
|
||||
Now func() time.Time
|
||||
@@ -317,6 +320,28 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
if table == "resources" {
|
||||
// 资源定位符里的桶必须换成本次写入的目标桶:字节就写在那里。
|
||||
raw, ok := row["physical_path"]
|
||||
if !ok {
|
||||
return errors.New("资源行缺少定位符")
|
||||
}
|
||||
var locator string
|
||||
if err := json.Unmarshal(raw, &locator); err != nil {
|
||||
return errors.New("资源定位符不可解析")
|
||||
}
|
||||
if t.config.ObjectsBucket != "" {
|
||||
rewritten, ok := rewriteLocatorBucket(locator, t.config.ObjectsBucket)
|
||||
if !ok {
|
||||
return fmt.Errorf("资源定位符无法改写到目标桶:%s", locator)
|
||||
}
|
||||
encoded, err := json.Marshal(rewritten)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
row["physical_path"] = encoded
|
||||
}
|
||||
}
|
||||
if columns == nil {
|
||||
columns = make([]string, 0, len(row))
|
||||
for key := range row {
|
||||
|
||||
@@ -100,6 +100,51 @@ func (importer *ObjectImporter) Promote(ctx context.Context, stage ObjectStage)
|
||||
return nil
|
||||
}
|
||||
|
||||
// rewriteLocatorBucket 把资源定位符里的桶换成目标桶,结构保持不变:
|
||||
// "storage://<backend>/<provider>://<source-bucket>/<key>" →
|
||||
// "storage://<backend>/<provider>://<target-bucket>/<key>"。
|
||||
//
|
||||
// 为什么必须改:同步把字节写进目标桶,但行是从快照原样搬的,定位符里写着测试桶;
|
||||
// 目标实例之后按这一行去取图/附件时会指向测试环境的桶(跨环境引用),
|
||||
// 或者干脆取不到。键不变,只换桶。
|
||||
func rewriteLocatorBucket(locator, bucket string) (string, bool) {
|
||||
if bucket == "" || locator == "" || locator != strings.TrimSpace(locator) {
|
||||
return "", false
|
||||
}
|
||||
prefix := ""
|
||||
rest := locator
|
||||
if strings.HasPrefix(locator, "storage://") {
|
||||
trimmed := strings.TrimPrefix(locator, "storage://")
|
||||
parts := strings.SplitN(trimmed, "/", 2)
|
||||
if len(parts) != 2 || !validSearchIdentifier(parts[0]) {
|
||||
return "", false
|
||||
}
|
||||
if parts[1] == "" {
|
||||
return "", false
|
||||
}
|
||||
prefix, rest = "storage://"+parts[0]+"/", parts[1]
|
||||
}
|
||||
separator := strings.Index(rest, "://")
|
||||
if separator < 0 {
|
||||
// 裸对象键里没有桶:落盘时用的是配置桶,不需要改,也不算失败。
|
||||
return locator, true
|
||||
}
|
||||
scheme := rest[:separator]
|
||||
if !validSearchIdentifier(scheme) {
|
||||
return "", false
|
||||
}
|
||||
schemeless := rest[separator+3:]
|
||||
boundary := strings.Index(schemeless, "/")
|
||||
if boundary <= 0 || boundary == len(schemeless)-1 {
|
||||
return "", false
|
||||
}
|
||||
key := schemeless[boundary+1:]
|
||||
if key == "" || !validSearchIdentifier(schemeless[:boundary]) {
|
||||
return "", false
|
||||
}
|
||||
return prefix + scheme + "://" + bucket + "/" + key, true
|
||||
}
|
||||
|
||||
func providerPathFromLocator(locator string) (string, bool) {
|
||||
rest := locator
|
||||
if strings.HasPrefix(locator, "storage://") {
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package v071
|
||||
|
||||
import "testing"
|
||||
|
||||
// 同步把字节写进目标桶,但资源行是从快照原样搬的:定位符里写着来源桶时,
|
||||
// 目标实例之后取图/附件会指向测试环境的桶(跨环境引用)或直接取不到。
|
||||
// 只换桶、不换键,两种定位符形状都要能改。
|
||||
func TestRewriteLocatorBucketKeepsKeyAndSwapsBucket(t *testing.T) {
|
||||
for _, testCase := range []struct{ locator, want string }{
|
||||
{"storage://45064e4f-434c-4752-a3ae-5fb1e2f54a7d/minio://kgmaint-1028a/10001/k/ab.md",
|
||||
"storage://45064e4f-434c-4752-a3ae-5fb1e2f54a7d/minio://kgseal-1029a/10001/k/ab.md"},
|
||||
{"oss://source-bucket/10/k/ab.png", "oss://target-bucket/10/k/ab.png"},
|
||||
} {
|
||||
bucket := "kgseal-1029a"
|
||||
if testCase.locator[:3] == "oss" {
|
||||
bucket = "target-bucket"
|
||||
}
|
||||
got, ok := rewriteLocatorBucket(testCase.locator, bucket)
|
||||
if !ok || got != testCase.want {
|
||||
t.Fatalf("rewrite(%q)=%q ok=%v want %q", testCase.locator, got, ok, testCase.want)
|
||||
}
|
||||
}
|
||||
// 解析不出来就报错,不猜:宁可失败也不能写一行指向别处的资源。
|
||||
for _, bad := range []string{"", "storage://x/", "storage://backend/minio://bucket", "oss://bucket/"} {
|
||||
if _, ok := rewriteLocatorBucket(bad, "target"); ok {
|
||||
t.Fatalf("非法定位符被接受了:%q", bad)
|
||||
}
|
||||
}
|
||||
if _, ok := rewriteLocatorBucket("oss://bucket/key", ""); ok {
|
||||
t.Fatal("没有目标桶时不该改写")
|
||||
}
|
||||
// 裸对象键里没有桶(落盘用配置桶),原样返回而不是失败。
|
||||
if got, ok := rewriteLocatorBucket("weknora/10000/k/ab.docx", "target"); !ok || got != "weknora/10000/k/ab.docx" {
|
||||
t.Fatalf("裸键被改写了:%q ok=%v", got, ok)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user