Files
knowledge-governance/cmd/worker/library_finish.go
T
姚章浩 8d2ed8ebe6
build / build (push) Successful in 1m32s
fix(library): 维护释放与服务恢复完成之后才暴露作业成功
原本 execute 写完生效版本就写 succeeded,而服务恢复、入口开放与维护锁释放发生在
窗口返回之后。任一步失败时外部已看到成功,重启接管又只认非终态,于是目标永远
留在维护里(本轮真实现象:同步 succeeded 后立即恢复被 409)。

现在内容校验与版本登记完成后只记"待收尾":finishLibrarySuccess 先确认目标写入者
与入口健康,再按 scope → instance 顺序释放两把锁(每一步错误都上抛,不再 `_ =` 丢弃),
最后才写 succeeded;任一步失败即落失败并保持 needs_restore。释放与终态写入用
context.WithoutCancel 有界包住,停机不会留下半成品。接管的 finalize-only 判定改为
按证据(finalize 阶段成功或未顶掉的生效版本),只复核与登记,不重写生产。

测试覆盖:调用顺序(成功后两把锁必已释放)、健康失败不释放不报成功、三步释放失败
各自不报成功、终态写入失败提示等待接管、接管证据判定正负例。
2026-10-29 14:02:09 +08:00

125 lines
6.2 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"errors"
"fmt"
"time"
"knowledge-governance/internal/governance"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// libraryMaintenanceStore 是"成功收尾"用到的存储能力。抽成接口是为了能在测试里
// 让某一步失败,验证"没全部完成就不许暴露成功"。
type libraryMaintenanceStore interface {
SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error
SetLibraryMaintenanceIngress(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error
UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error
UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error
ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error
ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
}
// releaseWindowLocks 是唯一释放路径。
//
// 顺序刻意是"scope 先、instance 后":instance 行是跨 scope 的互斥来源,最后放它
// 才能保证任何新作业在全部记录落定之前都进不来。每一步的错误都必须向上暴露——
// 只丢日志会让"锁其实没放"这件事变成页面上的成功。
func releaseWindowLocks(ctx context.Context, store libraryMaintenanceStore, instance, scope, jobID, note string, now time.Time) error {
if err := store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now); err != nil {
return fmt.Errorf("记录入口已开放失败(scope):%w", err)
}
if err := store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now); err != nil {
return fmt.Errorf("记录入口已开放失败(instance):%w", err)
}
if err := store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now); err != nil {
return fmt.Errorf("维护状态写入失败(scope):%w", err)
}
if err := store.ReleaseLibraryMaintenance(ctx, scope, jobID, now); err != nil {
return fmt.Errorf("维护占用释放失败(scope):%w", err)
}
if err := store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now); err != nil {
return fmt.Errorf("维护状态写入失败(instance):%w", err)
}
if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); err != nil {
return fmt.Errorf("实例占用释放失败(instance):%w", err)
}
return nil
}
// healthChecker 是"目标写入者与入口是否真的恢复健康"的判定口(测试可注入失败)。
type healthChecker interface {
VerifyRunning(ctx context.Context) error
}
// successBoundary 描述一次"内容已通过验证与版本登记,但还没到可以对外报成功"的作业。
type successBoundary struct {
Ready bool
Backup string
Target string
FinishedAt time.Time
}
// finishLibrarySuccess 是唯一的成功收尾入口:只有以下全部成立才暴露 succeeded——
// 目标服务健康、入口开放、scope 与 instance 两把维护锁都释放完成、终态写入成功。
//
// 为什么不能先标成功再释放:恢复/开入口/放锁都可能失败,先写 succeeded 会让外部
// (页面、API、重启后的接管逻辑)看到一条"成功了但目标仍在维护"的作业;而
// InterruptedLibraryJobs 只接管非终态,终态作业不会再被接管,锁就永远挂着。
func finishLibrarySuccess(ctx context.Context, store libraryMaintenanceStore, health healthChecker,
instance, scope, jobID string, boundary successBoundary) error {
// 释放与终态写入必须能在 ctx 取消后继续:停机不该留下"锁放了但作业还在跑"或
// 反过来"作业成功了锁还挂着"的任一半成品。
bounded, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second)
defer cancel()
holdForRepair := func(reason string) error {
note := "内容已验证,但收尾未完成,目标保持维护:" + reason
_ = store.UpdateLibraryMaintenanceState(bounded, scope, jobID, "needs_restore", note, time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(bounded, instance, jobID, "needs_restore", note, time.Now().UTC())
_ = store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobFailed, "finalize", note,
"生产内容已写入并校验;维护保持,按预备份或人工核验后再开放", boundary.Backup, boundary.Target, time.Now().UTC())
return errors.New(note)
}
if err := health.VerifyRunning(bounded); err != nil {
return holdForRepair("目标服务或入口未恢复健康:" + err.Error())
}
if err := releaseWindowLocks(bounded, store, instance, scope, jobID,
"写入与校验完成,目标已按原运行状态恢复", time.Now().UTC()); err != nil {
return holdForRepair(err.Error())
}
finishedAt := time.Now().UTC()
if err := store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobSucceeded, "finalize", "", "",
boundary.Backup, boundary.Target, finishedAt); err != nil {
// 锁已释放但终态没写下去:作业保持非终态,重启后会被接管(接管走
// finalize-only,只复核与登记,不重写生产)。
return fmt.Errorf("维护已释放但作业终态写入失败(等待接管):%w", err)
}
return nil
}
// needsFinalizeOnly 判断一个被接管的作业是否"内容与版本都已落定、只差收尾"。
// 判据只认真实证据:终态版本记录存在且未被顶掉,或 finalize 阶段已成功。
func needsFinalizeOnly(job governance.LibraryJob, versions []governance.LibraryVersion) bool {
if job.TargetSnapshotID == "" {
return false
}
for _, stage := range job.Stages {
if stage.Stage == "finalize" && stage.State == "succeeded" {
return true
}
}
for _, version := range versions {
if version.Kind == governance.LibraryVersionProductionEffective && version.SupersededAt == nil &&
version.SnapshotID == job.TargetSnapshotID {
return true
}
}
return false
}
var _ libraryMaintenanceStore = (*postgresstore.Store)(nil)