feat(library): 目标维护窗口(持久所有权/独占/release-on-verify)+ known-broken 记录
build / build (push) Failing after 16s

为什么:用户确认维护窗口方案,需要真实占有目标而不是 503 页面。

- 迁移 000070 library_maintenance:scope 唯一持有者、作业/目标身份、服务原状态、阶段。
- worker 在写目标前 acquire,验证+登记成功才 release;回滚失败/未验证保持 needs_restore。
- API GET /library/maintenance + 同步入口在他人持有时 409。
- 已知缺陷如实记录:候选实例该路由 404;kill 取证脚本竞态导致三项断言为 false。
This commit is contained in:
姚章浩
2026-10-28 20:45:02 +08:00
parent b76a433574
commit 288e01a01a
10 changed files with 250 additions and 6 deletions
+22
View File
@@ -114,6 +114,19 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup")
}
defer target.Close()
// 维护窗口:目标在同步/恢复期间被本作业独占,任何其他写入一律拒绝。
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity,
State: "acquired", OriginalServiceState: "candidate: no external writer configured",
Note: "同步/恢复期间目标维护,验证通过才开放",
}, now)
if acquireErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup")
}
_ = maintenance
if payload.FinalizeOnly {
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
@@ -142,6 +155,8 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
// 只有校验+登记都完成才开放目标。
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC())
return nil
}
@@ -155,6 +170,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
ParamsDigest: digest, Now: now,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
@@ -193,16 +209,22 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
// 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。
return fmt.Errorf("作业状态写入失败: %w", advanceErr)
}
// 验证通过 + 版本登记成功 = 唯一开放目标的路径。
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, finishedAt)
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID)
return nil
case "rolled_back":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"已回滚到上一版本,但验证未通过:目标继续维护,人工确认后再开放", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"回滚失败:目标保持维护,必须按预备份恢复", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
default:
recovery := outcome.Recovery
+16
View File
@@ -8,6 +8,7 @@ import (
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
@@ -54,6 +55,21 @@ func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail st
func int64Ptr(value int64) *int64 { return &value }
// redactDSN 只保留库名,避免把连接串写进维护记录。
func redactDSN(dsn string) string {
if dsn == "" {
return ""
}
if index := strings.LastIndex(dsn, "/"); index >= 0 && index+1 < len(dsn) {
name := dsn[index+1:]
if cut := strings.Index(name, "?"); cut >= 0 {
name = name[:cut]
}
return name
}
return "configured"
}
// stageSize 是某一类数据在快照里的真实总量(行数/字节)。
type stageSize struct {
count int64