Files
knowledge-governance/cmd/worker/library_finish.go
姚章浩 24d2546cbb
build / build (push) Successful in 1m28s
fix(library): 接管不再把版本查询失败当成没有收尾证据
resumed 分支里 LibraryVersions 的错误被忽略,FinalizeOnly 保持 false 后继续完整
apply:对"已登记版本并释放窗口、只差终态"的作业,这等于把同一份内容重写一遍目标,
与"最终只复核"的承诺相反。

现在接管先分类:finalize 阶段成功作为独立证据(不受版本查询影响)→ 只收尾;查询
失败且无阶段证据 → 未知,零写入、可重试;本快照被别的快照顶替 → 拒绝覆盖;其余才
按正常流程。

同时修释放幂等:Update*/Release* 对"已由本作业释放"的行返回 ErrMaintenanceReleased,
收尾重试视为已完成(部分释放后不会永久卡死),别的作业仍报未持有。真 PG 负例验证
两种情形。
2026-10-29 14:24:12 +08:00

163 lines
7.9 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"errors"
"fmt"
"time"
"knowledge-governance/internal/governance"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// libraryMaintenanceStore 是"成功收尾"用到的存储能力。抽成接口是为了能在测试里
// 让某一步失败,验证"没全部完成就不许暴露成功"。
type libraryMaintenanceStore interface {
SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error
SetLibraryMaintenanceIngress(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error
UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error
UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error
ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error
ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
}
// releaseWindowLocks 是唯一释放路径。
//
// 顺序刻意是"scope 先、instance 后":instance 行是跨 scope 的互斥来源,最后放它
// 才能保证任何新作业在全部记录落定之前都进不来。每一步的错误都必须向上暴露——
// 只丢日志会让"锁其实没放"这件事变成页面上的成功。
func releaseWindowLocks(ctx context.Context, store libraryMaintenanceStore, instance, scope, jobID, note string, now time.Time) error {
if err := store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now); !releasedOrNil(err) {
return fmt.Errorf("记录入口已开放失败(scope):%w", err)
}
if err := store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now); !releasedOrNil(err) {
return fmt.Errorf("记录入口已开放失败(instance):%w", err)
}
if err := store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now); !releasedOrNil(err) {
return fmt.Errorf("维护状态写入失败(scope):%w", err)
}
if err := store.ReleaseLibraryMaintenance(ctx, scope, jobID, now); !releasedOrNil(err) {
return fmt.Errorf("维护占用释放失败(scope):%w", err)
}
if err := store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now); !releasedOrNil(err) {
return fmt.Errorf("维护状态写入失败(instance):%w", err)
}
if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); !releasedOrNil(err) {
return fmt.Errorf("实例占用释放失败(instance):%w", err)
}
return nil
}
// healthChecker 是"目标写入者与入口是否真的恢复健康"的判定口(测试可注入失败)。
type healthChecker interface {
VerifyRunning(ctx context.Context) error
}
// successBoundary 描述一次"内容已通过验证与版本登记,但还没到可以对外报成功"的作业。
type successBoundary struct {
Ready bool
Backup string
Target string
FinishedAt time.Time
}
// finishLibrarySuccess 是唯一的成功收尾入口:只有以下全部成立才暴露 succeeded——
// 目标服务健康、入口开放、scope 与 instance 两把维护锁都释放完成、终态写入成功。
//
// 为什么不能先标成功再释放:恢复/开入口/放锁都可能失败,先写 succeeded 会让外部
// (页面、API、重启后的接管逻辑)看到一条"成功了但目标仍在维护"的作业;而
// InterruptedLibraryJobs 只接管非终态,终态作业不会再被接管,锁就永远挂着。
func finishLibrarySuccess(ctx context.Context, store libraryMaintenanceStore, health healthChecker,
instance, scope, jobID string, boundary successBoundary) error {
// 释放与终态写入必须能在 ctx 取消后继续:停机不该留下"锁放了但作业还在跑"或
// 反过来"作业成功了锁还挂着"的任一半成品。
bounded, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second)
defer cancel()
holdForRepair := func(reason string) error {
note := "内容已验证,但收尾未完成,目标保持维护:" + reason
_ = store.UpdateLibraryMaintenanceState(bounded, scope, jobID, "needs_restore", note, time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(bounded, instance, jobID, "needs_restore", note, time.Now().UTC())
_ = store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobFailed, "finalize", note,
"生产内容已写入并校验;维护保持,按预备份或人工核验后再开放", boundary.Backup, boundary.Target, time.Now().UTC())
return errors.New(note)
}
if err := health.VerifyRunning(bounded); err != nil {
return holdForRepair("目标服务或入口未恢复健康:" + err.Error())
}
if err := releaseWindowLocks(bounded, store, instance, scope, jobID,
"写入与校验完成,目标已按原运行状态恢复", time.Now().UTC()); err != nil {
return holdForRepair(err.Error())
}
finishedAt := time.Now().UTC()
if err := store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobSucceeded, "finalize", "", "",
boundary.Backup, boundary.Target, finishedAt); err != nil {
// 锁已释放但终态没写下去:作业保持非终态,重启后会被接管(接管走
// finalize-only,只复核与登记,不重写生产)。
return fmt.Errorf("维护已释放但作业终态写入失败(等待接管):%w", err)
}
return nil
}
// takeoverKind 是接管时"这个作业走到哪一步"的分类。
//
// 四类必须分开:把"版本查不到"当成"没有收尾证据",就会把只差收尾的作业重新写一遍
// 目标;把"曾被顶替"当成正常重放,就会用旧快照覆盖更新的生产版本。
type takeoverKind string
const (
// takeoverFresh:没有任何收尾证据,按正常流程(有预备份则复用,写前仍备份)。
takeoverFresh takeoverKind = "fresh"
// takeoverPendingFinal:内容与版本已落定,只差收尾(只复核与登记,不重写)。
takeoverPendingFinal takeoverKind = "pending_final"
// takeoverSuperseded:本快照曾经生效、后来被别的快照顶替,不得自动覆盖。
takeoverSuperseded takeoverKind = "superseded"
// takeoverUnknown:版本状态读不到且没有阶段证据,属"未知",不得写入。
takeoverUnknown takeoverKind = "unknown"
)
// classifyTakeover 只按证据分类:
// - finalize 阶段成功是独立证据(来自作业自己的阶段行),不因版本查询失败而丢弃;
// - 本快照存在 released/被顶替的版本 ⇒ superseded;
// - 本快照存在未被顶掉的生效版本 ⇒ pending_final;
// - 查询失败且无阶段证据 ⇒ unknown(可重试,不当成没有证据)。
func classifyTakeover(job governance.LibraryJob, versions []governance.LibraryVersion, versionsErr error) takeoverKind {
finalized := false
for _, stage := range job.Stages {
if stage.Stage == "finalize" && stage.State == "succeeded" {
finalized = true
}
}
if versionsErr != nil && !finalized {
return takeoverUnknown
}
superseded, effective := false, false
for _, version := range versions {
if version.Kind != governance.LibraryVersionProductionEffective || version.SnapshotID != job.TargetSnapshotID {
continue
}
if version.SupersededAt != nil {
superseded = true
} else {
effective = true
}
}
if superseded && !effective {
return takeoverSuperseded
}
if effective || finalized {
return takeoverPendingFinal
}
return takeoverFresh
}
var _ libraryMaintenanceStore = (*postgresstore.Store)(nil)
// releasedOrNil:这一行已经由本作业释放过,说明该步在上一次尝试里已经完成。
// 部分成功后重试幂等靠它——否则"scope 放了、instance 没放"的作业永远重试不下去。
func releasedOrNil(err error) bool {
return err == nil || errors.Is(err, postgresstore.ErrMaintenanceReleased)
}