diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c77d67..a3e2feb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,8 +31,15 @@ 字段等第一次写入才动态生成;首次同步因此在读取目标状态时就失败。现在"没有范围字段 + 确实 0 篇文档"按空基线处理,有文档却看不出归属仍然报错。 +- **资源定位符的桶没有跟着换**:同步把字节写进目标桶,但 `resources.physical_path` + 是从快照原样搬的,里面仍写着来源桶(`storage://…/minio://kgmaint-1028a/…`)。 + 目标实例之后取图/附件会指向测试环境的桶,或直接取不到。现在写入前把定位符里的桶 + 换成本次目标桶(键不变),解析不出来就报错而不是写一行指向别处的资源。 + ### Tests +- `internal/releaseadapter/v071/resource_locator_test.go`:嵌套/单层两种定位符换桶、裸键原样、 + 非法形状拒绝。 - `internal/releaseadapter/v071/search_empty_index_test.go`:空索引=空基线、有文档无范围字段必须报错。 - `internal/httpapi/library_capture_test.go`:队列作业 id 必须放得进 `varchar(36)` 且形状固定。 - `internal/releaseadapter/v071/search_tenant_filter_test.go`:接受未写租户、排除他租户、 diff --git a/_s3get/main.go b/_s3get/main.go new file mode 100644 index 0000000..de72563 --- /dev/null +++ b/_s3get/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// 列出并读回目标桶里的对象,打印 键/字节/sha256:资源字节是否真在目标存储里,只能这样核。 +func main() { + endpoint, bucket := os.Getenv("S3_ENDPOINT"), os.Getenv("S3_BUCKET") + client := s3.New(s3.Options{ + Region: "us-east-1", + BaseEndpoint: aws.String(endpoint), + Credentials: credentials.NewStaticCredentialsProvider(os.Getenv("S3_KEY"), os.Getenv("S3_SECRET"), ""), + UsePathStyle: true, + }) + ctx := context.Background() + pages := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{Bucket: aws.String(bucket)}) + for pages.HasMorePages() { + page, err := pages.NextPage(ctx) + if err != nil { + fmt.Println("list error:", err) + os.Exit(1) + } + for _, item := range page.Contents { + out, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: item.Key}) + if err != nil { + fmt.Println("get error:", item.Key, err) + os.Exit(1) + } + hash := sha256.New() + size, _ := io.Copy(hash, out.Body) + out.Body.Close() + fmt.Printf("%s\t%d\t%s\n", *item.Key, size, hex.EncodeToString(hash.Sum(nil))) + } + } +} \ No newline at end of file diff --git a/cmd/worker/library_wiring.go b/cmd/worker/library_wiring.go index 5be0d33..b4a6dfc 100644 --- a/cmd/worker/library_wiring.go +++ b/cmd/worker/library_wiring.go @@ -54,8 +54,9 @@ func buildLibraryProductionStack(ctx context.Context, config appconfig.Config, s target, err := v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{ Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL, SearchIndex: config.LibraryProductionIndex, Objects: objects, ObjectsReader: objects, - ArchiveRoot: config.LibraryArchiveRoot, - MaxDocuments: 200000, + ArchiveRoot: config.LibraryArchiveRoot, + ObjectsBucket: config.LibraryProductionS3Bucket, + MaxDocuments: 200000, }, identity) if err != nil { return nil, nil, err diff --git a/internal/releaseadapter/v071/library_production_target.go b/internal/releaseadapter/v071/library_production_target.go index 969ee5c..49fcb26 100644 --- a/internal/releaseadapter/v071/library_production_target.go +++ b/internal/releaseadapter/v071/library_production_target.go @@ -47,6 +47,9 @@ type ProductionLibraryConfig struct { Objects ObjectStore ObjectsReader ObjectReader ArchiveRoot string + // ObjectsBucket 是目标对象存储的桶名:资源行里的定位符要换成它, + // 否则同步过去的图/附件仍指向来源桶。 + ObjectsBucket string MaxDocuments int HTTPClient *http.Client Now func() time.Time @@ -317,6 +320,28 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc if len(row) == 0 { continue } + if table == "resources" { + // 资源定位符里的桶必须换成本次写入的目标桶:字节就写在那里。 + raw, ok := row["physical_path"] + if !ok { + return errors.New("资源行缺少定位符") + } + var locator string + if err := json.Unmarshal(raw, &locator); err != nil { + return errors.New("资源定位符不可解析") + } + if t.config.ObjectsBucket != "" { + rewritten, ok := rewriteLocatorBucket(locator, t.config.ObjectsBucket) + if !ok { + return fmt.Errorf("资源定位符无法改写到目标桶:%s", locator) + } + encoded, err := json.Marshal(rewritten) + if err != nil { + return err + } + row["physical_path"] = encoded + } + } if columns == nil { columns = make([]string, 0, len(row)) for key := range row { diff --git a/internal/releaseadapter/v071/object_import.go b/internal/releaseadapter/v071/object_import.go index 156d1ce..728215b 100644 --- a/internal/releaseadapter/v071/object_import.go +++ b/internal/releaseadapter/v071/object_import.go @@ -100,6 +100,51 @@ func (importer *ObjectImporter) Promote(ctx context.Context, stage ObjectStage) return nil } +// rewriteLocatorBucket 把资源定位符里的桶换成目标桶,结构保持不变: +// "storage:///:///" → +// "storage:///:///"。 +// +// 为什么必须改:同步把字节写进目标桶,但行是从快照原样搬的,定位符里写着测试桶; +// 目标实例之后按这一行去取图/附件时会指向测试环境的桶(跨环境引用), +// 或者干脆取不到。键不变,只换桶。 +func rewriteLocatorBucket(locator, bucket string) (string, bool) { + if bucket == "" || locator == "" || locator != strings.TrimSpace(locator) { + return "", false + } + prefix := "" + rest := locator + if strings.HasPrefix(locator, "storage://") { + trimmed := strings.TrimPrefix(locator, "storage://") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) != 2 || !validSearchIdentifier(parts[0]) { + return "", false + } + if parts[1] == "" { + return "", false + } + prefix, rest = "storage://"+parts[0]+"/", parts[1] + } + separator := strings.Index(rest, "://") + if separator < 0 { + // 裸对象键里没有桶:落盘时用的是配置桶,不需要改,也不算失败。 + return locator, true + } + scheme := rest[:separator] + if !validSearchIdentifier(scheme) { + return "", false + } + schemeless := rest[separator+3:] + boundary := strings.Index(schemeless, "/") + if boundary <= 0 || boundary == len(schemeless)-1 { + return "", false + } + key := schemeless[boundary+1:] + if key == "" || !validSearchIdentifier(schemeless[:boundary]) { + return "", false + } + return prefix + scheme + "://" + bucket + "/" + key, true +} + func providerPathFromLocator(locator string) (string, bool) { rest := locator if strings.HasPrefix(locator, "storage://") { diff --git a/internal/releaseadapter/v071/resource_locator_test.go b/internal/releaseadapter/v071/resource_locator_test.go new file mode 100644 index 0000000..5fdfaf8 --- /dev/null +++ b/internal/releaseadapter/v071/resource_locator_test.go @@ -0,0 +1,36 @@ +package v071 + +import "testing" + +// 同步把字节写进目标桶,但资源行是从快照原样搬的:定位符里写着来源桶时, +// 目标实例之后取图/附件会指向测试环境的桶(跨环境引用)或直接取不到。 +// 只换桶、不换键,两种定位符形状都要能改。 +func TestRewriteLocatorBucketKeepsKeyAndSwapsBucket(t *testing.T) { + for _, testCase := range []struct{ locator, want string }{ + {"storage://45064e4f-434c-4752-a3ae-5fb1e2f54a7d/minio://kgmaint-1028a/10001/k/ab.md", + "storage://45064e4f-434c-4752-a3ae-5fb1e2f54a7d/minio://kgseal-1029a/10001/k/ab.md"}, + {"oss://source-bucket/10/k/ab.png", "oss://target-bucket/10/k/ab.png"}, + } { + bucket := "kgseal-1029a" + if testCase.locator[:3] == "oss" { + bucket = "target-bucket" + } + got, ok := rewriteLocatorBucket(testCase.locator, bucket) + if !ok || got != testCase.want { + t.Fatalf("rewrite(%q)=%q ok=%v want %q", testCase.locator, got, ok, testCase.want) + } + } + // 解析不出来就报错,不猜:宁可失败也不能写一行指向别处的资源。 + for _, bad := range []string{"", "storage://x/", "storage://backend/minio://bucket", "oss://bucket/"} { + if _, ok := rewriteLocatorBucket(bad, "target"); ok { + t.Fatalf("非法定位符被接受了:%q", bad) + } + } + if _, ok := rewriteLocatorBucket("oss://bucket/key", ""); ok { + t.Fatal("没有目标桶时不该改写") + } + // 裸对象键里没有桶(落盘用配置桶),原样返回而不是失败。 + if got, ok := rewriteLocatorBucket("weknora/10000/k/ab.docx", "target"); !ok || got != "weknora/10000/k/ab.docx" { + t.Fatalf("裸键被改写了:%q ok=%v", got, ok) + } +}