fix(v071): 同步资源行时把定位符的桶换成目标桶
build / build (push) Has been cancelled

同步把资源字节写进目标桶,但 resources.physical_path 是从快照原样搬的,
里面还写着来源桶(storage://…/minio://kgmaint-1028a/…)。目标实例之后按这一行
取图/附件就会指向测试环境的桶,或直接取不到——跨环境引用,必须在本轮修掉:
写入前换桶、键不变,定位符解析不出来就报错(不写指向别处的行)。

试过:make verify 15 项全绿;新增定位符用例(嵌套/单层/裸键/非法);
真实同步后目标桶里对象键与行内定位符一致。
This commit is contained in:
姚章浩
2026-10-29 06:21:53 +08:00
parent 7d4ffdbfe5
commit cb0090802b
6 changed files with 161 additions and 2 deletions
+7
View File
@@ -31,8 +31,15 @@
字段等第一次写入才动态生成;首次同步因此在读取目标状态时就失败。现在"没有范围字段
+ 确实 0 篇文档"按空基线处理,有文档却看不出归属仍然报错。
- **资源定位符的桶没有跟着换**:同步把字节写进目标桶,但 `resources.physical_path`
是从快照原样搬的,里面仍写着来源桶(`storage://…/minio://kgmaint-1028a/…`)。
目标实例之后取图/附件会指向测试环境的桶,或直接取不到。现在写入前把定位符里的桶
换成本次目标桶(键不变),解析不出来就报错而不是写一行指向别处的资源。
### Tests
- `internal/releaseadapter/v071/resource_locator_test.go`:嵌套/单层两种定位符换桶、裸键原样、
非法形状拒绝。
- `internal/releaseadapter/v071/search_empty_index_test.go`:空索引=空基线、有文档无范围字段必须报错。
- `internal/httpapi/library_capture_test.go`:队列作业 id 必须放得进 `varchar(36)` 且形状固定。
- `internal/releaseadapter/v071/search_tenant_filter_test.go`:接受未写租户、排除他租户、
+45
View File
@@ -0,0 +1,45 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
// 列出并读回目标桶里的对象,打印 键/字节/sha256:资源字节是否真在目标存储里,只能这样核。
func main() {
endpoint, bucket := os.Getenv("S3_ENDPOINT"), os.Getenv("S3_BUCKET")
client := s3.New(s3.Options{
Region: "us-east-1",
BaseEndpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentialsProvider(os.Getenv("S3_KEY"), os.Getenv("S3_SECRET"), ""),
UsePathStyle: true,
})
ctx := context.Background()
pages := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{Bucket: aws.String(bucket)})
for pages.HasMorePages() {
page, err := pages.NextPage(ctx)
if err != nil {
fmt.Println("list error:", err)
os.Exit(1)
}
for _, item := range page.Contents {
out, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: item.Key})
if err != nil {
fmt.Println("get error:", item.Key, err)
os.Exit(1)
}
hash := sha256.New()
size, _ := io.Copy(hash, out.Body)
out.Body.Close()
fmt.Printf("%s\t%d\t%s\n", *item.Key, size, hex.EncodeToString(hash.Sum(nil)))
}
}
}
+3 -2
View File
@@ -54,8 +54,9 @@ func buildLibraryProductionStack(ctx context.Context, config appconfig.Config, s
target, err := v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{
Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL,
SearchIndex: config.LibraryProductionIndex, Objects: objects, ObjectsReader: objects,
ArchiveRoot: config.LibraryArchiveRoot,
MaxDocuments: 200000,
ArchiveRoot: config.LibraryArchiveRoot,
ObjectsBucket: config.LibraryProductionS3Bucket,
MaxDocuments: 200000,
}, identity)
if err != nil {
return nil, nil, err
@@ -47,6 +47,9 @@ type ProductionLibraryConfig struct {
Objects ObjectStore
ObjectsReader ObjectReader
ArchiveRoot string
// ObjectsBucket 是目标对象存储的桶名:资源行里的定位符要换成它,
// 否则同步过去的图/附件仍指向来源桶。
ObjectsBucket string
MaxDocuments int
HTTPClient *http.Client
Now func() time.Time
@@ -317,6 +320,28 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc
if len(row) == 0 {
continue
}
if table == "resources" {
// 资源定位符里的桶必须换成本次写入的目标桶:字节就写在那里。
raw, ok := row["physical_path"]
if !ok {
return errors.New("资源行缺少定位符")
}
var locator string
if err := json.Unmarshal(raw, &locator); err != nil {
return errors.New("资源定位符不可解析")
}
if t.config.ObjectsBucket != "" {
rewritten, ok := rewriteLocatorBucket(locator, t.config.ObjectsBucket)
if !ok {
return fmt.Errorf("资源定位符无法改写到目标桶:%s", locator)
}
encoded, err := json.Marshal(rewritten)
if err != nil {
return err
}
row["physical_path"] = encoded
}
}
if columns == nil {
columns = make([]string, 0, len(row))
for key := range row {
@@ -100,6 +100,51 @@ func (importer *ObjectImporter) Promote(ctx context.Context, stage ObjectStage)
return nil
}
// rewriteLocatorBucket 把资源定位符里的桶换成目标桶,结构保持不变:
// "storage://<backend>/<provider>://<source-bucket>/<key>" →
// "storage://<backend>/<provider>://<target-bucket>/<key>"。
//
// 为什么必须改:同步把字节写进目标桶,但行是从快照原样搬的,定位符里写着测试桶;
// 目标实例之后按这一行去取图/附件时会指向测试环境的桶(跨环境引用),
// 或者干脆取不到。键不变,只换桶。
func rewriteLocatorBucket(locator, bucket string) (string, bool) {
if bucket == "" || locator == "" || locator != strings.TrimSpace(locator) {
return "", false
}
prefix := ""
rest := locator
if strings.HasPrefix(locator, "storage://") {
trimmed := strings.TrimPrefix(locator, "storage://")
parts := strings.SplitN(trimmed, "/", 2)
if len(parts) != 2 || !validSearchIdentifier(parts[0]) {
return "", false
}
if parts[1] == "" {
return "", false
}
prefix, rest = "storage://"+parts[0]+"/", parts[1]
}
separator := strings.Index(rest, "://")
if separator < 0 {
// 裸对象键里没有桶:落盘时用的是配置桶,不需要改,也不算失败。
return locator, true
}
scheme := rest[:separator]
if !validSearchIdentifier(scheme) {
return "", false
}
schemeless := rest[separator+3:]
boundary := strings.Index(schemeless, "/")
if boundary <= 0 || boundary == len(schemeless)-1 {
return "", false
}
key := schemeless[boundary+1:]
if key == "" || !validSearchIdentifier(schemeless[:boundary]) {
return "", false
}
return prefix + scheme + "://" + bucket + "/" + key, true
}
func providerPathFromLocator(locator string) (string, bool) {
rest := locator
if strings.HasPrefix(locator, "storage://") {
@@ -0,0 +1,36 @@
package v071
import "testing"
// 同步把字节写进目标桶,但资源行是从快照原样搬的:定位符里写着来源桶时,
// 目标实例之后取图/附件会指向测试环境的桶(跨环境引用)或直接取不到。
// 只换桶、不换键,两种定位符形状都要能改。
func TestRewriteLocatorBucketKeepsKeyAndSwapsBucket(t *testing.T) {
for _, testCase := range []struct{ locator, want string }{
{"storage://45064e4f-434c-4752-a3ae-5fb1e2f54a7d/minio://kgmaint-1028a/10001/k/ab.md",
"storage://45064e4f-434c-4752-a3ae-5fb1e2f54a7d/minio://kgseal-1029a/10001/k/ab.md"},
{"oss://source-bucket/10/k/ab.png", "oss://target-bucket/10/k/ab.png"},
} {
bucket := "kgseal-1029a"
if testCase.locator[:3] == "oss" {
bucket = "target-bucket"
}
got, ok := rewriteLocatorBucket(testCase.locator, bucket)
if !ok || got != testCase.want {
t.Fatalf("rewrite(%q)=%q ok=%v want %q", testCase.locator, got, ok, testCase.want)
}
}
// 解析不出来就报错,不猜:宁可失败也不能写一行指向别处的资源。
for _, bad := range []string{"", "storage://x/", "storage://backend/minio://bucket", "oss://bucket/"} {
if _, ok := rewriteLocatorBucket(bad, "target"); ok {
t.Fatalf("非法定位符被接受了:%q", bad)
}
}
if _, ok := rewriteLocatorBucket("oss://bucket/key", ""); ok {
t.Fatal("没有目标桶时不该改写")
}
// 裸对象键里没有桶(落盘用配置桶),原样返回而不是失败。
if got, ok := rewriteLocatorBucket("weknora/10000/k/ab.docx", "target"); !ok || got != "weknora/10000/k/ab.docx" {
t.Fatalf("裸键被改写了:%q ok=%v", got, ok)
}
}