1aecd9d6af
build / build (push) Successful in 1m16s
storage://<id> 是来源实例 storage_backends 的主键;只换桶不改 id 时,目标原版
应用以 storage backend not found 落回默认服务并拒绝整条路径(真实下载 500
code1007)。写入前按租户+目标桶解析目标后端,写入与校验共用同一份行变换;
解析不到就拒绝写入,不再产出目标读不出来的定位符。
试过:目标实例真实同步后 /api/v1/knowledge/{id}/download 读回 500 → 修后复测;
go test ./internal/releaseadapter/v071 与 make verify 15 项全绿。
191 lines
7.1 KiB
Go
191 lines
7.1 KiB
Go
package v071
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"strings"
|
|
)
|
|
|
|
type ObjectStore interface {
|
|
Put(ctx context.Context, provider, path string, reader io.Reader, size int64) error
|
|
Stat(ctx context.Context, provider, path string) (size int64, sha256 string, err error)
|
|
Copy(ctx context.Context, provider, sourcePath, targetPath string) error
|
|
}
|
|
|
|
type TargetObjectConfig struct {
|
|
StorageBackendID string
|
|
Provider string
|
|
}
|
|
|
|
type StagedObject struct {
|
|
Handle string
|
|
StagingPath string
|
|
FinalPath string
|
|
Size int64
|
|
SHA256 string
|
|
}
|
|
|
|
type ObjectStage struct {
|
|
ReleaseRevision string
|
|
Provider string
|
|
Objects []StagedObject
|
|
}
|
|
|
|
type ObjectImporter struct {
|
|
store ObjectStore
|
|
config TargetObjectConfig
|
|
}
|
|
|
|
func NewObjectImporter(store ObjectStore, config TargetObjectConfig) (*ObjectImporter, error) {
|
|
config.StorageBackendID = strings.TrimSpace(config.StorageBackendID)
|
|
config.Provider = strings.TrimSpace(config.Provider)
|
|
if store == nil || !validSearchIdentifier(config.StorageBackendID) || !validObjectProvider(config.Provider) {
|
|
return nil, errors.New("invalid target object importer configuration")
|
|
}
|
|
return &ObjectImporter{store: store, config: config}, nil
|
|
}
|
|
|
|
func (importer *ObjectImporter) Stage(ctx context.Context, releaseRevision string, snapshot ObjectSnapshot) (ObjectStage, error) {
|
|
if !validSearchIdentifier(releaseRevision) || !validLowerSHA256(snapshot.SHA256) || snapshot.SHA256 != objectSnapshotDigest(snapshot.Objects) {
|
|
return ObjectStage{}, errors.New("invalid object import snapshot")
|
|
}
|
|
var total int64
|
|
stage := ObjectStage{ReleaseRevision: releaseRevision, Provider: importer.config.Provider, Objects: make([]StagedObject, 0, len(snapshot.Objects))}
|
|
for _, object := range snapshot.Objects {
|
|
if !validSearchIdentifier(object.ResourceID) || len(object.Handle) != 22 || object.Size != int64(len(object.Data)) || object.SHA256 != shaBytes(object.Data) || !validObjectKey(object.PhysicalPath) {
|
|
return ObjectStage{}, errors.New("invalid object import entry")
|
|
}
|
|
providerPath, ok := providerPathFromLocator(object.PhysicalPath)
|
|
if !ok {
|
|
return ObjectStage{}, errors.New("invalid object import locator")
|
|
}
|
|
total += object.Size
|
|
stagingPath := ".governance-release-stage/" + releaseRevision + "/" + providerPath
|
|
if !validObjectLocator(stagingPath) {
|
|
return ObjectStage{}, errors.New("invalid object staging locator")
|
|
}
|
|
if err := importer.store.Put(ctx, importer.config.Provider, stagingPath, bytes.NewReader(object.Data), object.Size); err != nil {
|
|
return ObjectStage{}, errors.New("write staged release object")
|
|
}
|
|
size, digest, err := importer.store.Stat(ctx, importer.config.Provider, stagingPath)
|
|
if err != nil || size != object.Size || digest != object.SHA256 {
|
|
return ObjectStage{}, errors.New("verify staged release object")
|
|
}
|
|
stage.Objects = append(stage.Objects, StagedObject{Handle: object.Handle, StagingPath: stagingPath, FinalPath: providerPath, Size: object.Size, SHA256: object.SHA256})
|
|
}
|
|
if total != snapshot.TotalBytes {
|
|
return ObjectStage{}, errors.New("object import total mismatch")
|
|
}
|
|
return stage, nil
|
|
}
|
|
|
|
func (importer *ObjectImporter) Promote(ctx context.Context, stage ObjectStage) error {
|
|
if stage.Provider != importer.config.Provider || !validSearchIdentifier(stage.ReleaseRevision) {
|
|
return errors.New("invalid object promotion stage")
|
|
}
|
|
for _, object := range stage.Objects {
|
|
if !validObjectLocator(object.StagingPath) || !validObjectLocator(object.FinalPath) || !validLowerSHA256(object.SHA256) || object.Size < 0 {
|
|
return errors.New("invalid object promotion entry")
|
|
}
|
|
if err := importer.store.Copy(ctx, importer.config.Provider, object.StagingPath, object.FinalPath); err != nil {
|
|
return errors.New("promote release object")
|
|
}
|
|
size, digest, err := importer.store.Stat(ctx, importer.config.Provider, object.FinalPath)
|
|
if err != nil || size != object.Size || digest != object.SHA256 {
|
|
return errors.New("verify promoted release object")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// rewriteLocatorBucket 把资源定位符里的桶换成目标桶,结构保持不变:
|
|
// "storage://<backend>/<provider>://<source-bucket>/<key>" →
|
|
// "storage://<backend>/<provider>://<target-bucket>/<key>"。
|
|
//
|
|
// 为什么必须改:同步把字节写进目标桶,但行是从快照原样搬的,定位符里写着测试桶;
|
|
// 目标实例之后按这一行去取图/附件时会指向测试环境的桶(跨环境引用),
|
|
// 或者干脆取不到。键不变,只换桶。
|
|
func rewriteLocatorBucket(locator, bucket string) (string, bool) {
|
|
return rewriteLocatorTarget(locator, "", bucket)
|
|
}
|
|
|
|
// rewriteLocatorTarget 同时改写定位符里的存储后端 id 与桶。
|
|
//
|
|
// 为什么后端 id 也要改:storage://<id>/minio://<bucket>/<key> 里的 id 是来源实例
|
|
// storage_backends 的主键。目标实例有自己的行;只改桶不改 id,原版应用会以
|
|
// "storage backend not found" 落到默认服务,然后拒绝整条路径(真跑 500 code1007)。
|
|
func rewriteLocatorTarget(locator, backendID, bucket string) (string, bool) {
|
|
if bucket == "" || locator == "" || locator != strings.TrimSpace(locator) {
|
|
return "", false
|
|
}
|
|
prefix := ""
|
|
rest := locator
|
|
if strings.HasPrefix(locator, "storage://") {
|
|
trimmed := strings.TrimPrefix(locator, "storage://")
|
|
parts := strings.SplitN(trimmed, "/", 2)
|
|
if len(parts) != 2 || !validSearchIdentifier(parts[0]) {
|
|
return "", false
|
|
}
|
|
if parts[1] == "" {
|
|
return "", false
|
|
}
|
|
if backendID != "" {
|
|
if !validSearchIdentifier(backendID) {
|
|
return "", false
|
|
}
|
|
parts[0] = backendID
|
|
}
|
|
prefix, rest = "storage://"+parts[0]+"/", parts[1]
|
|
}
|
|
separator := strings.Index(rest, "://")
|
|
if separator < 0 {
|
|
// 裸对象键里没有桶:落盘时用的是配置桶,不需要改,也不算失败。
|
|
return locator, true
|
|
}
|
|
scheme := rest[:separator]
|
|
if !validSearchIdentifier(scheme) {
|
|
return "", false
|
|
}
|
|
schemeless := rest[separator+3:]
|
|
boundary := strings.Index(schemeless, "/")
|
|
if boundary <= 0 || boundary == len(schemeless)-1 {
|
|
return "", false
|
|
}
|
|
key := schemeless[boundary+1:]
|
|
if key == "" || !validSearchIdentifier(schemeless[:boundary]) {
|
|
return "", false
|
|
}
|
|
return prefix + scheme + "://" + bucket + "/" + key, true
|
|
}
|
|
|
|
func providerPathFromLocator(locator string) (string, bool) {
|
|
rest := locator
|
|
if strings.HasPrefix(locator, "storage://") {
|
|
rest = strings.TrimPrefix(locator, "storage://")
|
|
parts := strings.SplitN(rest, "/", 2)
|
|
if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validObjectLocator(parts[1]) {
|
|
return "", false
|
|
}
|
|
rest = parts[1]
|
|
} else if !validObjectLocator(locator) {
|
|
return "", false
|
|
}
|
|
// WeKnora nests the provider locator inside the storage locator:
|
|
// "storage://<backend-id>/oss://<bucket>/<key>". S3 keys address the
|
|
// bucket configured for the store, so strip the inner scheme and bucket.
|
|
if separator := strings.Index(rest, "://"); separator >= 0 {
|
|
if !validSearchIdentifier(rest[:separator]) {
|
|
return "", false
|
|
}
|
|
schemeless := rest[separator+3:]
|
|
boundary := strings.Index(schemeless, "/")
|
|
if boundary < 0 || boundary == len(schemeless)-1 {
|
|
return "", false
|
|
}
|
|
rest = schemeless[boundary+1:]
|
|
}
|
|
return rest, validObjectLocator(rest)
|
|
}
|