删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。 新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下 执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。 预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者; 写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
This commit is contained in:
@@ -42,6 +42,9 @@ type libraryVersionView struct {
|
||||
// librarySyncJobType 与 worker 的处理器类型一致;单独定义避免 API 依赖 worker 包。
|
||||
const librarySyncJobType = "library.sync"
|
||||
|
||||
// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型(同一份定义也在 worker 里)。
|
||||
const libraryRecoverJobType = "library.recover"
|
||||
|
||||
type libraryKnownView struct {
|
||||
State string `json:"state"` // known | unknown
|
||||
Version *libraryVersionView `json:"version,omitempty"`
|
||||
@@ -291,6 +294,101 @@ func libraryStore(config Config) (libraryJobStore, bool) {
|
||||
return store, ok
|
||||
}
|
||||
|
||||
// instanceMaintenanceGuard 在"实例级维护"被别的作业持有时拒绝新的写入请求。
|
||||
// 只看 scope 行是不够的:internal 与 external 动的是同一批容器。
|
||||
func instanceMaintenanceGuard(ctx context.Context, config Config, scope string) (map[string]any, bool) {
|
||||
instance := config.LibraryTargetInstance
|
||||
if instance == "" {
|
||||
return nil, false
|
||||
}
|
||||
store, ok := config.Store.(interface {
|
||||
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
|
||||
})
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
held, err := store.InstanceMaintenanceOf(ctx, instance)
|
||||
if err != nil || held == nil || held.ReleasedAt != nil {
|
||||
return nil, false
|
||||
}
|
||||
return map[string]any{"code": "library_target_in_maintenance",
|
||||
"error": "目标实例正在维护中,验证完成前不接受新的写入请求",
|
||||
"instance": held.Instance, "owner_job": held.OwnerJobID, "owner_scope": held.OwnerScope,
|
||||
"state": held.State, "ingress_state": held.IngressState}, true
|
||||
}
|
||||
|
||||
// libraryMaintenanceRecoverHandler 是"按预备份恢复并校验"的唯一入口:
|
||||
// 它只建一个恢复作业,真正的恢复与校验由 worker 在维护关闭状态下执行。
|
||||
func libraryMaintenanceRecoverHandler(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||||
return
|
||||
}
|
||||
var request struct {
|
||||
Scope string `json:"scope"`
|
||||
WindowJobID string `json:"window_job_id"`
|
||||
}
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return
|
||||
}
|
||||
if request.Scope != "internal" && request.Scope != "external" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||||
return
|
||||
}
|
||||
instance := config.LibraryTargetInstance
|
||||
if instance == "" {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_recovery_unavailable",
|
||||
"error": "目标维护未接线:无法执行按预备份恢复"})
|
||||
return
|
||||
}
|
||||
store, ok := config.Store.(interface {
|
||||
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
|
||||
})
|
||||
if !ok {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "维护记录存储不可用"})
|
||||
return
|
||||
}
|
||||
held, err := store.InstanceMaintenanceOf(r.Context(), instance)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护记录"})
|
||||
return
|
||||
}
|
||||
if held == nil || held.ReleasedAt != nil {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_no_held_window",
|
||||
"error": "目标实例当前没有未释放的维护窗口,不需要恢复"})
|
||||
return
|
||||
}
|
||||
if held.OwnerScope != request.Scope {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_scope_mismatch",
|
||||
"error": "维护窗口属于另一个 scope", "owner_scope": held.OwnerScope})
|
||||
return
|
||||
}
|
||||
if request.WindowJobID != "" && request.WindowJobID != held.OwnerJobID {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_mismatch",
|
||||
"error": "维护窗口属于另一个作业", "owner_job": held.OwnerJobID})
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
payload, err := json.Marshal(map[string]string{"scope": held.OwnerScope, "window_job_id": held.OwnerJobID})
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build recovery payload"})
|
||||
return
|
||||
}
|
||||
// 队列作业 id 与幂等键都按窗口固定:重复点击只会得到同一个恢复作业。
|
||||
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||||
ID: "librecover-" + held.OwnerJobID, Type: libraryRecoverJobType, Payload: payload,
|
||||
AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.recover:" + held.OwnerJobID,
|
||||
}, now)
|
||||
if err != nil || queueJob == nil {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_recovery_enqueue_failed",
|
||||
"error": "恢复作业投递失败(或已有同窗口的恢复作业)", "detail": errString(err)})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": queueJob, "window_job_id": held.OwnerJobID,
|
||||
"scope": held.OwnerScope, "instance": held.Instance})
|
||||
}
|
||||
|
||||
// activeGrantFor 只查"目标+快照+同一参数摘要"的未过期未撤销授权。
|
||||
func activeGrantFor(ctx context.Context, config Config, scope, snapshotID string) (governance.LibraryReleaseAuthorization, bool) {
|
||||
if snapshotID == "" {
|
||||
@@ -597,6 +695,11 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
}
|
||||
grantFound, hasGrant := activeGrantFor(r.Context(), config, request.Scope, request.Target)
|
||||
_ = grantFound
|
||||
// 目标实例正在维护(跨对内/对外互斥):直接拒绝,不等到 worker 里才发现。
|
||||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||||
writeJSON(w, http.StatusConflict, body)
|
||||
return
|
||||
}
|
||||
// 目标正在维护(被别的作业持有):直接拒绝新写入,不等到 worker 里才发现。
|
||||
if maintenanceStore, ok := config.Store.(interface {
|
||||
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error)
|
||||
|
||||
@@ -54,6 +54,10 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
|
||||
return
|
||||
}
|
||||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||||
writeJSON(w, http.StatusConflict, body)
|
||||
return
|
||||
}
|
||||
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||||
|
||||
@@ -145,6 +145,9 @@ type Config struct {
|
||||
VersionHistoryRepo string
|
||||
// LibraryArchiveRoot 是整库封存产物的只读归档根;留空即未接线(页面降级)。
|
||||
LibraryArchiveRoot string
|
||||
// LibraryTargetInstance 是实例级维护的目标实例名;留空即"未接线",
|
||||
// 页面不能发起按预备份恢复,接口返回原因而不是假装能恢复。
|
||||
LibraryTargetInstance string
|
||||
// LibraryAgentSocket 是主机侧封存 agent 的 unix socket;留空即页面不能发起封存。
|
||||
LibraryAgentSocket string
|
||||
// LibraryCapacityBudgetBytes 是整库归档容量预算;0 = 未设置(容量门禁不放行)。
|
||||
@@ -461,7 +464,22 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护窗口"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held})
|
||||
response := map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held}
|
||||
// 实例级占用才是真正的互斥来源:跨 scope 也在这里体现。
|
||||
if instanceStore, ok := config.Store.(interface {
|
||||
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
|
||||
}); ok && config.LibraryTargetInstance != "" {
|
||||
if instanceHeld, err := instanceStore.InstanceMaintenanceOf(r.Context(), config.LibraryTargetInstance); err == nil {
|
||||
response["instance"] = config.LibraryTargetInstance
|
||||
response["instance_maintenance"] = instanceHeld
|
||||
response["held"] = instanceHeld != nil && instanceHeld.ReleasedAt == nil
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
})
|
||||
// 按预备份恢复并逐类校验:维护窗口卡住时唯一的收尾入口(worker 执行)。
|
||||
mux.HandleFunc("POST /api/v1/library/maintenance/recover", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryMaintenanceRecoverHandler(w, r, config)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/library/jobs", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryJobsHandler(w, r, config)
|
||||
|
||||
Reference in New Issue
Block a user