fix(library): 维护收尾必须按预备份恢复并校验
build / build (push) Successful in 1m20s

删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。
新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下
执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。
预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者;
写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
This commit is contained in:
姚章浩
2026-10-29 00:50:32 +08:00
parent ec737a9a65
commit 3643e9bf82
26 changed files with 1730 additions and 225 deletions
+103
View File
@@ -42,6 +42,9 @@ type libraryVersionView struct {
// librarySyncJobType 与 worker 的处理器类型一致;单独定义避免 API 依赖 worker 包。
const librarySyncJobType = "library.sync"
// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型(同一份定义也在 worker 里)。
const libraryRecoverJobType = "library.recover"
type libraryKnownView struct {
State string `json:"state"` // known | unknown
Version *libraryVersionView `json:"version,omitempty"`
@@ -291,6 +294,101 @@ func libraryStore(config Config) (libraryJobStore, bool) {
return store, ok
}
// instanceMaintenanceGuard 在"实例级维护"被别的作业持有时拒绝新的写入请求。
// 只看 scope 行是不够的:internal 与 external 动的是同一批容器。
func instanceMaintenanceGuard(ctx context.Context, config Config, scope string) (map[string]any, bool) {
instance := config.LibraryTargetInstance
if instance == "" {
return nil, false
}
store, ok := config.Store.(interface {
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
})
if !ok {
return nil, false
}
held, err := store.InstanceMaintenanceOf(ctx, instance)
if err != nil || held == nil || held.ReleasedAt != nil {
return nil, false
}
return map[string]any{"code": "library_target_in_maintenance",
"error": "目标实例正在维护中,验证完成前不接受新的写入请求",
"instance": held.Instance, "owner_job": held.OwnerJobID, "owner_scope": held.OwnerScope,
"state": held.State, "ingress_state": held.IngressState}, true
}
// libraryMaintenanceRecoverHandler 是"按预备份恢复并校验"的唯一入口:
// 它只建一个恢复作业,真正的恢复与校验由 worker 在维护关闭状态下执行。
func libraryMaintenanceRecoverHandler(w http.ResponseWriter, r *http.Request, config Config) {
if r.Method != http.MethodPost {
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
return
}
var request struct {
Scope string `json:"scope"`
WindowJobID string `json:"window_job_id"`
}
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
return
}
if request.Scope != "internal" && request.Scope != "external" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
return
}
instance := config.LibraryTargetInstance
if instance == "" {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_recovery_unavailable",
"error": "目标维护未接线:无法执行按预备份恢复"})
return
}
store, ok := config.Store.(interface {
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
})
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "维护记录存储不可用"})
return
}
held, err := store.InstanceMaintenanceOf(r.Context(), instance)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护记录"})
return
}
if held == nil || held.ReleasedAt != nil {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_no_held_window",
"error": "目标实例当前没有未释放的维护窗口,不需要恢复"})
return
}
if held.OwnerScope != request.Scope {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_scope_mismatch",
"error": "维护窗口属于另一个 scope", "owner_scope": held.OwnerScope})
return
}
if request.WindowJobID != "" && request.WindowJobID != held.OwnerJobID {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_mismatch",
"error": "维护窗口属于另一个作业", "owner_job": held.OwnerJobID})
return
}
now := time.Now().UTC()
payload, err := json.Marshal(map[string]string{"scope": held.OwnerScope, "window_job_id": held.OwnerJobID})
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build recovery payload"})
return
}
// 队列作业 id 与幂等键都按窗口固定:重复点击只会得到同一个恢复作业。
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: "librecover-" + held.OwnerJobID, Type: libraryRecoverJobType, Payload: payload,
AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.recover:" + held.OwnerJobID,
}, now)
if err != nil || queueJob == nil {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_recovery_enqueue_failed",
"error": "恢复作业投递失败(或已有同窗口的恢复作业)", "detail": errString(err)})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": queueJob, "window_job_id": held.OwnerJobID,
"scope": held.OwnerScope, "instance": held.Instance})
}
// activeGrantFor 只查"目标+快照+同一参数摘要"的未过期未撤销授权。
func activeGrantFor(ctx context.Context, config Config, scope, snapshotID string) (governance.LibraryReleaseAuthorization, bool) {
if snapshotID == "" {
@@ -597,6 +695,11 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
}
grantFound, hasGrant := activeGrantFor(r.Context(), config, request.Scope, request.Target)
_ = grantFound
// 目标实例正在维护(跨对内/对外互斥):直接拒绝,不等到 worker 里才发现。
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
writeJSON(w, http.StatusConflict, body)
return
}
// 目标正在维护(被别的作业持有):直接拒绝新写入,不等到 worker 里才发现。
if maintenanceStore, ok := config.Store.(interface {
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error)
+4
View File
@@ -54,6 +54,10 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
return
}
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
writeJSON(w, http.StatusConflict, body)
return
}
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
+19 -1
View File
@@ -145,6 +145,9 @@ type Config struct {
VersionHistoryRepo string
// LibraryArchiveRoot 是整库封存产物的只读归档根;留空即未接线(页面降级)。
LibraryArchiveRoot string
// LibraryTargetInstance 是实例级维护的目标实例名;留空即"未接线",
// 页面不能发起按预备份恢复,接口返回原因而不是假装能恢复。
LibraryTargetInstance string
// LibraryAgentSocket 是主机侧封存 agent 的 unix socket;留空即页面不能发起封存。
LibraryAgentSocket string
// LibraryCapacityBudgetBytes 是整库归档容量预算;0 = 未设置(容量门禁不放行)。
@@ -461,7 +464,22 @@ func NewHandler(config Config) (http.Handler, error) {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护窗口"})
return
}
writeJSON(w, http.StatusOK, map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held})
response := map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held}
// 实例级占用才是真正的互斥来源:跨 scope 也在这里体现。
if instanceStore, ok := config.Store.(interface {
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
}); ok && config.LibraryTargetInstance != "" {
if instanceHeld, err := instanceStore.InstanceMaintenanceOf(r.Context(), config.LibraryTargetInstance); err == nil {
response["instance"] = config.LibraryTargetInstance
response["instance_maintenance"] = instanceHeld
response["held"] = instanceHeld != nil && instanceHeld.ReleasedAt == nil
}
}
writeJSON(w, http.StatusOK, response)
})
// 按预备份恢复并逐类校验:维护窗口卡住时唯一的收尾入口(worker 执行)。
mux.HandleFunc("POST /api/v1/library/maintenance/recover", func(w http.ResponseWriter, r *http.Request) {
libraryMaintenanceRecoverHandler(w, r, config)
})
mux.HandleFunc("GET /api/v1/library/jobs", func(w http.ResponseWriter, r *http.Request) {
libraryJobsHandler(w, r, config)