feat(library): 目标维护窗口(持久所有权/独占/release-on-verify)+ known-broken 记录
build / build (push) Failing after 16s
build / build (push) Failing after 16s
为什么:用户确认维护窗口方案,需要真实占有目标而不是 503 页面。 - 迁移 000070 library_maintenance:scope 唯一持有者、作业/目标身份、服务原状态、阶段。 - worker 在写目标前 acquire,验证+登记成功才 release;回滚失败/未验证保持 needs_restore。 - API GET /library/maintenance + 同步入口在他人持有时 409。 - 已知缺陷如实记录:候选实例该路由 404;kill 取证脚本竞态导致三项断言为 false。
This commit is contained in:
@@ -114,6 +114,19 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup")
|
||||
}
|
||||
defer target.Close()
|
||||
// 维护窗口:目标在同步/恢复期间被本作业独占,任何其他写入一律拒绝。
|
||||
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
|
||||
redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
|
||||
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
|
||||
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity,
|
||||
State: "acquired", OriginalServiceState: "candidate: no external writer configured",
|
||||
Note: "同步/恢复期间目标维护,验证通过才开放",
|
||||
}, now)
|
||||
if acquireErr != nil {
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup")
|
||||
}
|
||||
_ = maintenance
|
||||
|
||||
if payload.FinalizeOnly {
|
||||
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
|
||||
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
|
||||
@@ -142,6 +155,8 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
|
||||
previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||||
// 只有校验+登记都完成才开放目标。
|
||||
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC())
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -155,6 +170,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
ParamsDigest: digest, Now: now,
|
||||
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
|
||||
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
|
||||
outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder)
|
||||
finishedAt := time.Now().UTC()
|
||||
switch outcome.State {
|
||||
@@ -193,16 +209,22 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
// 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。
|
||||
return fmt.Errorf("作业状态写入失败: %w", advanceErr)
|
||||
}
|
||||
// 验证通过 + 版本登记成功 = 唯一开放目标的路径。
|
||||
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, finishedAt)
|
||||
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
|
||||
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID)
|
||||
return nil
|
||||
case "rolled_back":
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
|
||||
"已回滚到上一版本,但验证未通过:目标继续维护,人工确认后再开放", finishedAt)
|
||||
return errors.New(outcome.Error)
|
||||
case "rollback_failed":
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
|
||||
"回滚失败:目标保持维护,必须按预备份恢复", finishedAt)
|
||||
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
|
||||
default:
|
||||
recovery := outcome.Recovery
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -54,6 +55,21 @@ func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail st
|
||||
|
||||
func int64Ptr(value int64) *int64 { return &value }
|
||||
|
||||
// redactDSN 只保留库名,避免把连接串写进维护记录。
|
||||
func redactDSN(dsn string) string {
|
||||
if dsn == "" {
|
||||
return ""
|
||||
}
|
||||
if index := strings.LastIndex(dsn, "/"); index >= 0 && index+1 < len(dsn) {
|
||||
name := dsn[index+1:]
|
||||
if cut := strings.Index(name, "?"); cut >= 0 {
|
||||
name = name[:cut]
|
||||
}
|
||||
return name
|
||||
}
|
||||
return "configured"
|
||||
}
|
||||
|
||||
// stageSize 是某一类数据在快照里的真实总量(行数/字节)。
|
||||
type stageSize struct {
|
||||
count int64
|
||||
|
||||
Reference in New Issue
Block a user