8d2ed8ebe6
build / build (push) Successful in 1m32s
原本 execute 写完生效版本就写 succeeded,而服务恢复、入口开放与维护锁释放发生在 窗口返回之后。任一步失败时外部已看到成功,重启接管又只认非终态,于是目标永远 留在维护里(本轮真实现象:同步 succeeded 后立即恢复被 409)。 现在内容校验与版本登记完成后只记"待收尾":finishLibrarySuccess 先确认目标写入者 与入口健康,再按 scope → instance 顺序释放两把锁(每一步错误都上抛,不再 `_ =` 丢弃), 最后才写 succeeded;任一步失败即落失败并保持 needs_restore。释放与终态写入用 context.WithoutCancel 有界包住,停机不会留下半成品。接管的 finalize-only 判定改为 按证据(finalize 阶段成功或未顶掉的生效版本),只复核与登记,不重写生产。 测试覆盖:调用顺序(成功后两把锁必已释放)、健康失败不释放不报成功、三步释放失败 各自不报成功、终态写入失败提示等待接管、接管证据判定正负例。
125 lines
6.2 KiB
Go
125 lines
6.2 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
)
|
||
|
||
// libraryMaintenanceStore 是"成功收尾"用到的存储能力。抽成接口是为了能在测试里
|
||
// 让某一步失败,验证"没全部完成就不许暴露成功"。
|
||
type libraryMaintenanceStore interface {
|
||
SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error
|
||
SetLibraryMaintenanceIngress(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error
|
||
UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error
|
||
UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error
|
||
ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error
|
||
ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error
|
||
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
|
||
}
|
||
|
||
// releaseWindowLocks 是唯一释放路径。
|
||
//
|
||
// 顺序刻意是"scope 先、instance 后":instance 行是跨 scope 的互斥来源,最后放它
|
||
// 才能保证任何新作业在全部记录落定之前都进不来。每一步的错误都必须向上暴露——
|
||
// 只丢日志会让"锁其实没放"这件事变成页面上的成功。
|
||
func releaseWindowLocks(ctx context.Context, store libraryMaintenanceStore, instance, scope, jobID, note string, now time.Time) error {
|
||
if err := store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now); err != nil {
|
||
return fmt.Errorf("记录入口已开放失败(scope):%w", err)
|
||
}
|
||
if err := store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now); err != nil {
|
||
return fmt.Errorf("记录入口已开放失败(instance):%w", err)
|
||
}
|
||
if err := store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now); err != nil {
|
||
return fmt.Errorf("维护状态写入失败(scope):%w", err)
|
||
}
|
||
if err := store.ReleaseLibraryMaintenance(ctx, scope, jobID, now); err != nil {
|
||
return fmt.Errorf("维护占用释放失败(scope):%w", err)
|
||
}
|
||
if err := store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now); err != nil {
|
||
return fmt.Errorf("维护状态写入失败(instance):%w", err)
|
||
}
|
||
if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); err != nil {
|
||
return fmt.Errorf("实例占用释放失败(instance):%w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// healthChecker 是"目标写入者与入口是否真的恢复健康"的判定口(测试可注入失败)。
|
||
type healthChecker interface {
|
||
VerifyRunning(ctx context.Context) error
|
||
}
|
||
|
||
// successBoundary 描述一次"内容已通过验证与版本登记,但还没到可以对外报成功"的作业。
|
||
type successBoundary struct {
|
||
Ready bool
|
||
Backup string
|
||
Target string
|
||
FinishedAt time.Time
|
||
}
|
||
|
||
// finishLibrarySuccess 是唯一的成功收尾入口:只有以下全部成立才暴露 succeeded——
|
||
// 目标服务健康、入口开放、scope 与 instance 两把维护锁都释放完成、终态写入成功。
|
||
//
|
||
// 为什么不能先标成功再释放:恢复/开入口/放锁都可能失败,先写 succeeded 会让外部
|
||
// (页面、API、重启后的接管逻辑)看到一条"成功了但目标仍在维护"的作业;而
|
||
// InterruptedLibraryJobs 只接管非终态,终态作业不会再被接管,锁就永远挂着。
|
||
func finishLibrarySuccess(ctx context.Context, store libraryMaintenanceStore, health healthChecker,
|
||
instance, scope, jobID string, boundary successBoundary) error {
|
||
// 释放与终态写入必须能在 ctx 取消后继续:停机不该留下"锁放了但作业还在跑"或
|
||
// 反过来"作业成功了锁还挂着"的任一半成品。
|
||
bounded, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second)
|
||
defer cancel()
|
||
|
||
holdForRepair := func(reason string) error {
|
||
note := "内容已验证,但收尾未完成,目标保持维护:" + reason
|
||
_ = store.UpdateLibraryMaintenanceState(bounded, scope, jobID, "needs_restore", note, time.Now().UTC())
|
||
_ = store.UpdateInstanceMaintenanceState(bounded, instance, jobID, "needs_restore", note, time.Now().UTC())
|
||
_ = store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobFailed, "finalize", note,
|
||
"生产内容已写入并校验;维护保持,按预备份或人工核验后再开放", boundary.Backup, boundary.Target, time.Now().UTC())
|
||
return errors.New(note)
|
||
}
|
||
|
||
if err := health.VerifyRunning(bounded); err != nil {
|
||
return holdForRepair("目标服务或入口未恢复健康:" + err.Error())
|
||
}
|
||
if err := releaseWindowLocks(bounded, store, instance, scope, jobID,
|
||
"写入与校验完成,目标已按原运行状态恢复", time.Now().UTC()); err != nil {
|
||
return holdForRepair(err.Error())
|
||
}
|
||
finishedAt := time.Now().UTC()
|
||
if err := store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobSucceeded, "finalize", "", "",
|
||
boundary.Backup, boundary.Target, finishedAt); err != nil {
|
||
// 锁已释放但终态没写下去:作业保持非终态,重启后会被接管(接管走
|
||
// finalize-only,只复核与登记,不重写生产)。
|
||
return fmt.Errorf("维护已释放但作业终态写入失败(等待接管):%w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// needsFinalizeOnly 判断一个被接管的作业是否"内容与版本都已落定、只差收尾"。
|
||
// 判据只认真实证据:终态版本记录存在且未被顶掉,或 finalize 阶段已成功。
|
||
func needsFinalizeOnly(job governance.LibraryJob, versions []governance.LibraryVersion) bool {
|
||
if job.TargetSnapshotID == "" {
|
||
return false
|
||
}
|
||
for _, stage := range job.Stages {
|
||
if stage.Stage == "finalize" && stage.State == "succeeded" {
|
||
return true
|
||
}
|
||
}
|
||
for _, version := range versions {
|
||
if version.Kind == governance.LibraryVersionProductionEffective && version.SupersededAt == nil &&
|
||
version.SnapshotID == job.TargetSnapshotID {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
var _ libraryMaintenanceStore = (*postgresstore.Store)(nil)
|