24d2546cbb
build / build (push) Successful in 1m28s
resumed 分支里 LibraryVersions 的错误被忽略,FinalizeOnly 保持 false 后继续完整 apply:对"已登记版本并释放窗口、只差终态"的作业,这等于把同一份内容重写一遍目标, 与"最终只复核"的承诺相反。 现在接管先分类:finalize 阶段成功作为独立证据(不受版本查询影响)→ 只收尾;查询 失败且无阶段证据 → 未知,零写入、可重试;本快照被别的快照顶替 → 拒绝覆盖;其余才 按正常流程。 同时修释放幂等:Update*/Release* 对"已由本作业释放"的行返回 ErrMaintenanceReleased, 收尾重试视为已完成(部分释放后不会永久卡死),别的作业仍报未持有。真 PG 负例验证 两种情形。
163 lines
7.9 KiB
Go
163 lines
7.9 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
)
|
||
|
||
// libraryMaintenanceStore 是"成功收尾"用到的存储能力。抽成接口是为了能在测试里
|
||
// 让某一步失败,验证"没全部完成就不许暴露成功"。
|
||
type libraryMaintenanceStore interface {
|
||
SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error
|
||
SetLibraryMaintenanceIngress(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error
|
||
UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error
|
||
UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error
|
||
ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error
|
||
ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error
|
||
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
|
||
}
|
||
|
||
// releaseWindowLocks 是唯一释放路径。
|
||
//
|
||
// 顺序刻意是"scope 先、instance 后":instance 行是跨 scope 的互斥来源,最后放它
|
||
// 才能保证任何新作业在全部记录落定之前都进不来。每一步的错误都必须向上暴露——
|
||
// 只丢日志会让"锁其实没放"这件事变成页面上的成功。
|
||
func releaseWindowLocks(ctx context.Context, store libraryMaintenanceStore, instance, scope, jobID, note string, now time.Time) error {
|
||
if err := store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now); !releasedOrNil(err) {
|
||
return fmt.Errorf("记录入口已开放失败(scope):%w", err)
|
||
}
|
||
if err := store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now); !releasedOrNil(err) {
|
||
return fmt.Errorf("记录入口已开放失败(instance):%w", err)
|
||
}
|
||
if err := store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now); !releasedOrNil(err) {
|
||
return fmt.Errorf("维护状态写入失败(scope):%w", err)
|
||
}
|
||
if err := store.ReleaseLibraryMaintenance(ctx, scope, jobID, now); !releasedOrNil(err) {
|
||
return fmt.Errorf("维护占用释放失败(scope):%w", err)
|
||
}
|
||
if err := store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now); !releasedOrNil(err) {
|
||
return fmt.Errorf("维护状态写入失败(instance):%w", err)
|
||
}
|
||
if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); !releasedOrNil(err) {
|
||
return fmt.Errorf("实例占用释放失败(instance):%w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// healthChecker 是"目标写入者与入口是否真的恢复健康"的判定口(测试可注入失败)。
|
||
type healthChecker interface {
|
||
VerifyRunning(ctx context.Context) error
|
||
}
|
||
|
||
// successBoundary 描述一次"内容已通过验证与版本登记,但还没到可以对外报成功"的作业。
|
||
type successBoundary struct {
|
||
Ready bool
|
||
Backup string
|
||
Target string
|
||
FinishedAt time.Time
|
||
}
|
||
|
||
// finishLibrarySuccess 是唯一的成功收尾入口:只有以下全部成立才暴露 succeeded——
|
||
// 目标服务健康、入口开放、scope 与 instance 两把维护锁都释放完成、终态写入成功。
|
||
//
|
||
// 为什么不能先标成功再释放:恢复/开入口/放锁都可能失败,先写 succeeded 会让外部
|
||
// (页面、API、重启后的接管逻辑)看到一条"成功了但目标仍在维护"的作业;而
|
||
// InterruptedLibraryJobs 只接管非终态,终态作业不会再被接管,锁就永远挂着。
|
||
func finishLibrarySuccess(ctx context.Context, store libraryMaintenanceStore, health healthChecker,
|
||
instance, scope, jobID string, boundary successBoundary) error {
|
||
// 释放与终态写入必须能在 ctx 取消后继续:停机不该留下"锁放了但作业还在跑"或
|
||
// 反过来"作业成功了锁还挂着"的任一半成品。
|
||
bounded, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second)
|
||
defer cancel()
|
||
|
||
holdForRepair := func(reason string) error {
|
||
note := "内容已验证,但收尾未完成,目标保持维护:" + reason
|
||
_ = store.UpdateLibraryMaintenanceState(bounded, scope, jobID, "needs_restore", note, time.Now().UTC())
|
||
_ = store.UpdateInstanceMaintenanceState(bounded, instance, jobID, "needs_restore", note, time.Now().UTC())
|
||
_ = store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobFailed, "finalize", note,
|
||
"生产内容已写入并校验;维护保持,按预备份或人工核验后再开放", boundary.Backup, boundary.Target, time.Now().UTC())
|
||
return errors.New(note)
|
||
}
|
||
|
||
if err := health.VerifyRunning(bounded); err != nil {
|
||
return holdForRepair("目标服务或入口未恢复健康:" + err.Error())
|
||
}
|
||
if err := releaseWindowLocks(bounded, store, instance, scope, jobID,
|
||
"写入与校验完成,目标已按原运行状态恢复", time.Now().UTC()); err != nil {
|
||
return holdForRepair(err.Error())
|
||
}
|
||
finishedAt := time.Now().UTC()
|
||
if err := store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobSucceeded, "finalize", "", "",
|
||
boundary.Backup, boundary.Target, finishedAt); err != nil {
|
||
// 锁已释放但终态没写下去:作业保持非终态,重启后会被接管(接管走
|
||
// finalize-only,只复核与登记,不重写生产)。
|
||
return fmt.Errorf("维护已释放但作业终态写入失败(等待接管):%w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// takeoverKind 是接管时"这个作业走到哪一步"的分类。
|
||
//
|
||
// 四类必须分开:把"版本查不到"当成"没有收尾证据",就会把只差收尾的作业重新写一遍
|
||
// 目标;把"曾被顶替"当成正常重放,就会用旧快照覆盖更新的生产版本。
|
||
type takeoverKind string
|
||
|
||
const (
|
||
// takeoverFresh:没有任何收尾证据,按正常流程(有预备份则复用,写前仍备份)。
|
||
takeoverFresh takeoverKind = "fresh"
|
||
// takeoverPendingFinal:内容与版本已落定,只差收尾(只复核与登记,不重写)。
|
||
takeoverPendingFinal takeoverKind = "pending_final"
|
||
// takeoverSuperseded:本快照曾经生效、后来被别的快照顶替,不得自动覆盖。
|
||
takeoverSuperseded takeoverKind = "superseded"
|
||
// takeoverUnknown:版本状态读不到且没有阶段证据,属"未知",不得写入。
|
||
takeoverUnknown takeoverKind = "unknown"
|
||
)
|
||
|
||
// classifyTakeover 只按证据分类:
|
||
// - finalize 阶段成功是独立证据(来自作业自己的阶段行),不因版本查询失败而丢弃;
|
||
// - 本快照存在 released/被顶替的版本 ⇒ superseded;
|
||
// - 本快照存在未被顶掉的生效版本 ⇒ pending_final;
|
||
// - 查询失败且无阶段证据 ⇒ unknown(可重试,不当成没有证据)。
|
||
func classifyTakeover(job governance.LibraryJob, versions []governance.LibraryVersion, versionsErr error) takeoverKind {
|
||
finalized := false
|
||
for _, stage := range job.Stages {
|
||
if stage.Stage == "finalize" && stage.State == "succeeded" {
|
||
finalized = true
|
||
}
|
||
}
|
||
if versionsErr != nil && !finalized {
|
||
return takeoverUnknown
|
||
}
|
||
superseded, effective := false, false
|
||
for _, version := range versions {
|
||
if version.Kind != governance.LibraryVersionProductionEffective || version.SnapshotID != job.TargetSnapshotID {
|
||
continue
|
||
}
|
||
if version.SupersededAt != nil {
|
||
superseded = true
|
||
} else {
|
||
effective = true
|
||
}
|
||
}
|
||
if superseded && !effective {
|
||
return takeoverSuperseded
|
||
}
|
||
if effective || finalized {
|
||
return takeoverPendingFinal
|
||
}
|
||
return takeoverFresh
|
||
}
|
||
|
||
var _ libraryMaintenanceStore = (*postgresstore.Store)(nil)
|
||
|
||
// releasedOrNil:这一行已经由本作业释放过,说明该步在上一次尝试里已经完成。
|
||
// 部分成功后重试幂等靠它——否则"scope 放了、instance 没放"的作业永远重试不下去。
|
||
func releasedOrNil(err error) bool {
|
||
return err == nil || errors.Is(err, postgresstore.ErrMaintenanceReleased)
|
||
}
|