3643e9bf82
build / build (push) Successful in 1m20s
删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。 新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下 执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。 预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者; 写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
406 lines
13 KiB
Go
406 lines
13 KiB
Go
package libraryfreeze
|
||
|
||
import (
|
||
"context"
|
||
"encoding/hex"
|
||
"errors"
|
||
"fmt"
|
||
"os"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
// Window 是一次维护窗口里"写目标"阶段的句柄。
|
||
//
|
||
// work 必须在第一笔目标写入之前调用 MarkContentRisk:它是一条持久证据,
|
||
// 记录"从这里开始目标内容可能已被改动"。进程在这之后死掉时,恢复器不会
|
||
// 自动把目标开放——半写目标只能由原作业接管或按预备份恢复。
|
||
type Window struct {
|
||
c *Coordinator
|
||
j *journal
|
||
}
|
||
|
||
func (w *Window) MarkContentRisk(ctx context.Context) error {
|
||
if w == nil || w.c == nil || w.j == nil {
|
||
return errors.New("invalid maintenance window")
|
||
}
|
||
if err := ctx.Err(); err != nil {
|
||
return err
|
||
}
|
||
if w.j.ContentRisk {
|
||
return nil
|
||
}
|
||
w.j.ContentRisk = true
|
||
return w.c.save(*w.j)
|
||
}
|
||
|
||
// SetBackupSnapshot 把"这次窗口的预备份身份"落盘。恢复只认它:
|
||
// 没有这份身份的窗口,之后既不能自动恢复,也不许被外部收尾。
|
||
func (w *Window) SetBackupSnapshot(ctx context.Context, snapshotID string) error {
|
||
if w == nil || w.c == nil || w.j == nil {
|
||
return errors.New("invalid maintenance window")
|
||
}
|
||
if err := ctx.Err(); err != nil {
|
||
return err
|
||
}
|
||
if !validDigest(snapshotID) {
|
||
return errors.New("invalid backup snapshot id")
|
||
}
|
||
if w.j.BackupSnapshotID != "" && w.j.BackupSnapshotID != snapshotID {
|
||
return errors.New("backup snapshot changed inside the same window")
|
||
}
|
||
w.j.BackupSnapshotID = snapshotID
|
||
return w.c.save(*w.j)
|
||
}
|
||
|
||
// Job 返回窗口所属作业 id(恢复器与调用方据此对账)。
|
||
func (w *Window) Job() string {
|
||
if w == nil || w.j == nil {
|
||
return ""
|
||
}
|
||
return w.j.Job
|
||
}
|
||
|
||
func (c *Coordinator) Run(ctx context.Context, job string, capture func(context.Context) (string, error)) (Result, error) {
|
||
return c.RunPrepared(ctx, job, func(context.Context) error { return nil }, capture)
|
||
}
|
||
|
||
// RunPrepared 是封存链路的入口:停止写入者 → 在窗口内取快照 → 恢复写入者。
|
||
// 它等价于 allowRestore 恒为 true 的 RunWindow。
|
||
func (c *Coordinator) RunPrepared(ctx context.Context, job string, prepare func(context.Context) error, capture func(context.Context) (string, error)) (Result, error) {
|
||
if capture == nil {
|
||
return Result{}, errors.New("invalid freeze request")
|
||
}
|
||
return c.RunWindow(ctx, job, prepare, func(ctx context.Context, _ *Window) (string, error) {
|
||
return capture(ctx)
|
||
}, func(error) bool { return true })
|
||
}
|
||
|
||
// RunWindow 是维护窗口的唯一实现:
|
||
//
|
||
// 关入口 + 停全部写入者 → 验证真的停了、在途真的排空了 → 跑 work
|
||
// → 恢复"原本在跑"的服务 → 验证入口回来
|
||
//
|
||
// allowRestore 在 work 失败后决定目标能否重新开放:返回 false 表示内容状态
|
||
// 未验证(半写)——此时服务保持停止、journal 保持非终态,崩溃恢复也不许
|
||
// 自动启动它们。同一个 job 可以接管未完成的窗口,别的 job 不行。
|
||
func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(context.Context) error,
|
||
work func(context.Context, *Window) (string, error), allowRestore func(error) bool) (Result, error) {
|
||
if !validID(job) || work == nil || prepare == nil {
|
||
return Result{}, errors.New("invalid freeze request")
|
||
}
|
||
if allowRestore == nil {
|
||
allowRestore = func(error) bool { return true }
|
||
}
|
||
if err := ctx.Err(); err != nil {
|
||
return Result{}, err
|
||
}
|
||
lock, err := c.lock()
|
||
if err != nil {
|
||
return Result{}, err
|
||
}
|
||
defer unlock(lock)
|
||
|
||
previous, loadErr := c.load(job)
|
||
j := journal{}
|
||
switch {
|
||
case loadErr == nil && previous.State == "succeeded":
|
||
return previous.Result, nil
|
||
case loadErr == nil && (previous.State == "stopping" || previous.State == "held"):
|
||
// 接管:目标必须仍然停住。不重新"准备"、也不从零重来。
|
||
j = previous
|
||
if j.State == "held" && j.RestoreBlocked {
|
||
// 上一次窗口把内容判成半写:接管方必须自己重新证明,不许清掉风险。
|
||
j.RestoreBlocked = false
|
||
}
|
||
case loadErr == nil:
|
||
return previous.Result, errors.New("freeze request already exists; recovery or a new request is required")
|
||
case !errors.Is(loadErr, os.ErrNotExist):
|
||
return Result{}, loadErr
|
||
default:
|
||
if err = c.pending(); err != nil {
|
||
return Result{}, err
|
||
}
|
||
if err = prepare(ctx); err != nil {
|
||
return Result{}, err
|
||
}
|
||
if err = ctx.Err(); err != nil {
|
||
return Result{}, err
|
||
}
|
||
items, err := inventory(ctx, c.control)
|
||
if err != nil {
|
||
return Result{}, err
|
||
}
|
||
j = journal{Version: 1, Source: c.source, Job: job, Level: c.level, Ingress: c.ingress,
|
||
TargetFingerprint: c.fingerprint, Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
|
||
for _, s := range items {
|
||
if s.Running && !s.Healthy {
|
||
return Result{}, errors.New("test writer is not ready")
|
||
}
|
||
j.Services = append(j.Services, serviceRecord{Service: s})
|
||
}
|
||
if err = c.save(j); err != nil {
|
||
return Result{}, err
|
||
}
|
||
}
|
||
|
||
// 后台任务必须先在"还能跑"的时候跑完:入口排空只覆盖 HTTP 连接,
|
||
// app 内的 asynq worker 会被上游的 exit(1) 直接切断。
|
||
if idleErr := c.awaitBackgroundIdle(ctx, &j); idleErr != nil {
|
||
return c.abandonWindow(ctx, &j, idleErr, "background_busy")
|
||
}
|
||
|
||
// 顺序是有原因的:入口先停(新请求被拒、在途请求跑完),写者后停。
|
||
// 反过来做,正在处理的请求就会被 SIGTERM 切断——那不算排空。
|
||
if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) {
|
||
evidence, gateErr := c.gate.CloseIngress(ctx)
|
||
if gateErr != nil {
|
||
return c.abandonWindow(ctx, &j, fmt.Errorf("关闭目标入口失败:%w", gateErr), "gate_failed")
|
||
}
|
||
if !evidence.Closed || !evidence.Drained {
|
||
return c.abandonWindow(ctx, &j, errors.New("目标入口未确认排空(仍有在途请求)"), "gate_not_drained")
|
||
}
|
||
j.Gate = &evidence
|
||
if err = c.save(j); err != nil {
|
||
return Result{}, err
|
||
}
|
||
}
|
||
|
||
// 停写入者:任何一步不成立都不许往下写。
|
||
if stopErr := c.stopAll(ctx, &j); stopErr != nil {
|
||
return c.abandonWindow(ctx, &j, stopErr, "stop_failed")
|
||
}
|
||
if stopErr := c.stopped(ctx, j); stopErr != nil {
|
||
return c.abandonWindow(ctx, &j, stopErr, "isolation_unverified")
|
||
}
|
||
if stopErr := c.verifyDrain(ctx, &j); stopErr != nil {
|
||
return c.abandonWindow(ctx, &j, stopErr, "drain_unverified")
|
||
}
|
||
if c.ingressClosed != nil {
|
||
if stopErr := c.ingressClosed(ctx); stopErr != nil {
|
||
return c.abandonWindow(ctx, &j, fmt.Errorf("目标入口仍然可用:%w", stopErr), "ingress_open")
|
||
}
|
||
j.IngressClosed = true
|
||
}
|
||
j.State = "held"
|
||
if err = c.save(j); err != nil {
|
||
return Result{}, err
|
||
}
|
||
|
||
w := &Window{c: c, j: &j}
|
||
// work 里 panic 等同于失败:目标可能已被改过,必须走恢复分支,不能让
|
||
// 进程直接崩掉把写入者留在停止状态。
|
||
snapshotID, workErr := func() (id string, err error) {
|
||
defer func() {
|
||
if recover() != nil {
|
||
id, err = "", errors.New("维护窗口内的作业 panic")
|
||
}
|
||
}()
|
||
return work(ctx, w)
|
||
}()
|
||
if workErr != nil {
|
||
return c.closeWindow(ctx, &j, workErr, allowRestore(workErr), "")
|
||
}
|
||
if err = ctx.Err(); err != nil {
|
||
// 取消/超时发生在窗口内:内容状态未知,但写入者必须恢复(旧行为一致),
|
||
// 由上层把作业判失败;不把"没验证完"说成成功。
|
||
return c.closeWindow(ctx, &j, err, true, "")
|
||
}
|
||
if snapshotID != "" {
|
||
if !validDigest(snapshotID) {
|
||
return c.closeWindow(ctx, &j, errors.New("invalid candidate snapshot digest"), true, "")
|
||
}
|
||
}
|
||
// work 报成功但写入者已经不在停住状态 = 隔离被破坏,不能登记成功。
|
||
if err = c.stopped(ctx, j); err != nil {
|
||
return c.closeWindow(ctx, &j, err, true, "")
|
||
}
|
||
return c.closeWindow(ctx, &j, nil, true, snapshotID)
|
||
}
|
||
|
||
// abandonWindow:在"还没写目标"的阶段失败——内容没被改过,可以安全恢复并结案。
|
||
func (c *Coordinator) abandonWindow(ctx context.Context, j *journal, cause error, code string) (Result, error) {
|
||
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
|
||
defer cancel()
|
||
if restoreErr := c.restore(recoveryCtx, j); restoreErr != nil {
|
||
// 恢复不了就保持非终态:下一次恢复/接管还要处理它。
|
||
j.Code = code
|
||
_ = c.save(*j)
|
||
return j.Result, errors.Join(cause, restoreErr)
|
||
}
|
||
ended := time.Now().UTC()
|
||
j.State = "failed"
|
||
j.Code = code
|
||
j.FinishedAt = &ended
|
||
if err := c.save(*j); err != nil {
|
||
return j.Result, errors.Join(cause, err)
|
||
}
|
||
return j.Result, cause
|
||
}
|
||
|
||
// closeWindow:写完之后的收尾。allowRestore=false(半写且回滚失败)时
|
||
// 服务保持停止、journal 保持非终态,返回的是"目标仍在维护"这个结论。
|
||
func (c *Coordinator) closeWindow(ctx context.Context, j *journal, cause error, allowRestore bool, snapshotID string) (Result, error) {
|
||
if snapshotID != "" {
|
||
j.SnapshotID = snapshotID
|
||
}
|
||
if !allowRestore {
|
||
j.State = "held"
|
||
j.RestoreBlocked = true
|
||
if cause == nil {
|
||
cause = errors.New("目标内容状态未验证")
|
||
}
|
||
j.Code = "restore_blocked"
|
||
if err := c.save(*j); err != nil {
|
||
return j.Result, errors.Join(cause, err)
|
||
}
|
||
return j.Result, cause
|
||
}
|
||
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
|
||
defer cancel()
|
||
if restoreErr := c.restore(recoveryCtx, j); restoreErr != nil {
|
||
// 服务没能恢复 = 入口没回来 = 目标仍在维护,不许报成功。
|
||
j.State = "held"
|
||
j.Code = "restore_failed"
|
||
if saveErr := c.save(*j); saveErr != nil {
|
||
return j.Result, errors.Join(cause, restoreErr, saveErr)
|
||
}
|
||
return j.Result, errors.Join(cause, restoreErr)
|
||
}
|
||
if c.ingressReady != nil {
|
||
if readyErr := c.ingressReady(recoveryCtx); readyErr != nil {
|
||
j.State = "held"
|
||
j.Code = "ingress_unverified"
|
||
_ = c.save(*j)
|
||
return j.Result, errors.Join(cause, fmt.Errorf("目标入口没有恢复:%w", readyErr))
|
||
}
|
||
}
|
||
ended := time.Now().UTC()
|
||
j.FinishedAt = &ended
|
||
if cause != nil {
|
||
j.State = "failed"
|
||
if j.Code == "" {
|
||
j.Code = "work_failed"
|
||
}
|
||
} else {
|
||
j.State = "succeeded"
|
||
j.Code = ""
|
||
}
|
||
if err := c.save(*j); err != nil {
|
||
return j.Result, errors.Join(cause, err)
|
||
}
|
||
return j.Result, cause
|
||
}
|
||
|
||
// awaitBackgroundIdle 等到后台任务队列为空再往下走。拿不到结论(查询失败)
|
||
// 也按"没空"处理:不能把"问不到"当成"没任务"。
|
||
func (c *Coordinator) awaitBackgroundIdle(ctx context.Context, j *journal) error {
|
||
if c.quiescer == nil {
|
||
if c.level == "instance" {
|
||
return errors.New("没有后台任务静默检查:不能证明停写者时后台任务已结束")
|
||
}
|
||
return nil
|
||
}
|
||
wait := c.quiesceWait
|
||
if wait <= 0 {
|
||
wait = 2 * time.Minute
|
||
}
|
||
deadline := time.Now().Add(wait)
|
||
var last string
|
||
for {
|
||
idle, detail, err := c.quiescer.Idle(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("读不到后台任务状态:%w", err)
|
||
}
|
||
if idle {
|
||
j.BackgroundIdle = true
|
||
return nil
|
||
}
|
||
last = detail
|
||
if time.Now().After(deadline) {
|
||
return fmt.Errorf("后台任务仍在运行(%s):等它结束再开窗口,否则会被 SIGTERM 切断", last)
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return ctx.Err()
|
||
case <-time.After(500 * time.Millisecond):
|
||
}
|
||
}
|
||
}
|
||
|
||
// stopAll 只对"原本在跑"的服务发停止指令,每发一个都先落盘意图:
|
||
// 进程在停止过程中死掉时,磁盘上仍能看出"哪些服务被要求停过"。
|
||
func (c *Coordinator) stopAll(ctx context.Context, j *journal) error {
|
||
var failures error
|
||
for i := range j.Services {
|
||
if !j.Services[i].Running || j.Services[i].StopRequested {
|
||
continue
|
||
}
|
||
j.Services[i].StopRequested = true
|
||
if err := c.save(*j); err != nil {
|
||
return err
|
||
}
|
||
if err := c.control.Stop(ctx, j.Services[i].ID); err != nil {
|
||
failures = errors.Join(failures, err)
|
||
break
|
||
}
|
||
}
|
||
return failures
|
||
}
|
||
|
||
// verifyDrain 用控制器报告的退出码判断"在途请求是否排空":
|
||
// 0 = 服务自己收尾退出;其它(143/137)= 被切断,不许当成排空。
|
||
// 声明了实例级维护时,拿不到退出码就是没有证据——按未排空处理。
|
||
func (c *Coordinator) verifyDrain(ctx context.Context, j *journal) error {
|
||
needed := 0
|
||
for _, s := range j.Services {
|
||
if s.Running && s.StopRequested {
|
||
needed++
|
||
}
|
||
}
|
||
if needed == 0 {
|
||
return nil
|
||
}
|
||
// 入口层已经证明"在途请求全部结束"时,写者的退出码只作为记录:
|
||
// 被代理的 WeKnora 收到 SIGTERM 会自己 exit(1)(upstream bug),
|
||
// 那不代表请求被切断——请求早在入口关闭时就已经跑完了。
|
||
gateDrained := j.Gate != nil && j.Gate.Closed && j.Gate.Drained
|
||
items, err := inventory(ctx, c.control)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
actual := map[string]Service{}
|
||
for _, s := range items {
|
||
actual[s.ID] = s
|
||
}
|
||
verified := 0
|
||
for i := range j.Services {
|
||
s := &j.Services[i]
|
||
if !s.Running || !s.StopRequested {
|
||
continue
|
||
}
|
||
now, ok := actual[s.ID]
|
||
if !ok || now.Running {
|
||
return errors.New("test writer still running")
|
||
}
|
||
if now.ExitCode == nil {
|
||
if c.level == "instance" && !gateDrained {
|
||
return fmt.Errorf("无法证明 %s 的在途请求已排空(控制器没有退出码)", s.ID)
|
||
}
|
||
continue
|
||
}
|
||
s.ExitCode = now.ExitCode
|
||
if *now.ExitCode != 0 && !gateDrained {
|
||
return fmt.Errorf("写入者 %s 未优雅退出(exit=%d):在途请求没有完成排空", s.ID, *now.ExitCode)
|
||
}
|
||
verified++
|
||
}
|
||
j.DrainVerified = gateDrained || verified == needed
|
||
return nil
|
||
}
|
||
|
||
func validDigest(id string) bool {
|
||
decoded, err := hex.DecodeString(id)
|
||
return err == nil && len(decoded) == 32 && strings.ToLower(id) == id
|
||
}
|