780ab988c4
build / build (push) Successful in 1m20s
真实现场 2026-10-30T00:19Z:新恢复作业恢复历史快照 fc595511(已被 0d982f76 顶替),写入 database 后被 SIGKILL;重启接管时 classifyTakeover 只要看到 该快照有 SupersededAt 就判 superseded,没有判这行历史版本属于哪个作业, 把历史那次发布被顶替当成这次作业过期,作业 failed/prebackup,而恢复文案 又写"无需恢复",目标半写且维护锁未释放。 修法:被顶替分支先分清归属——本作业 finalize 成功或版本行属于本作业 ⇒ 仍 拒绝;历史行属于别的作业时,仅当本次是 restore、授权签发于顶替之后、scope 与 instance 两把锁仍由本作业独占且指纹一致、预备份登记属于本作业且清单字节 仍等于摘要,才按 fresh 继续重跑整条链路。任一证据缺失/读不到仍拒绝。 同时把续作早退统一走 failLibraryJobOrTakeover:有持久化预备份时保留 needs_restore 并点名 backup,不再写"无需恢复"。 验证: - 先写失败回归(历史行+新授权 ⇒ 期望 fresh,实得 superseded),修后绿; - 真实 PostgreSQL 正负例:授权撤销/实例锁释放/预备份归属被改/清单字节被改 均回到 superseded;续作早退保留备份与 needs_restore; - 本地 make verify 16 项必需检查全绿(identity 本地档案跳过)。
248 lines
11 KiB
Go
248 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
postgresstore "knowledge-governance/internal/platform/postgres"
|
|
)
|
|
|
|
type fakeFinishStore struct {
|
|
calls []string
|
|
failOn string
|
|
alreadyReleased bool
|
|
advanced []governance.LibraryJobState
|
|
// states 记录所有维护状态写入(测试要区分 verified 与 needs_restore)。
|
|
states []string
|
|
}
|
|
|
|
// 模拟"这一行已由同一个作业释放过"(真实 Store 返回 ErrMaintenanceReleased)。
|
|
func (f *fakeFinishStore) releasedSentinel() error {
|
|
if f.alreadyReleased {
|
|
return postgresstore.ErrMaintenanceReleased
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeFinishStore) record(name string) error {
|
|
f.calls = append(f.calls, name)
|
|
if f.failOn == name {
|
|
return errors.New("injected " + name + " failure")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeFinishStore) SetInstanceMaintenanceIngress(context.Context, string, string, string, string, time.Time) error {
|
|
if err := f.releasedSentinel(); err != nil {
|
|
return err
|
|
}
|
|
return f.record("instance_ingress")
|
|
}
|
|
func (f *fakeFinishStore) SetLibraryMaintenanceIngress(context.Context, string, string, string, string, time.Time) error {
|
|
if err := f.releasedSentinel(); err != nil {
|
|
return err
|
|
}
|
|
return f.record("scope_ingress")
|
|
}
|
|
func (f *fakeFinishStore) UpdateInstanceMaintenanceState(_ context.Context, _, _, state, _ string, _ time.Time) error {
|
|
if err := f.releasedSentinel(); err != nil {
|
|
return err
|
|
}
|
|
f.states = append(f.states, state)
|
|
return f.record("instance_state")
|
|
}
|
|
func (f *fakeFinishStore) UpdateLibraryMaintenanceState(_ context.Context, _, _, state, _ string, _ time.Time) error {
|
|
if err := f.releasedSentinel(); err != nil {
|
|
return err
|
|
}
|
|
f.states = append(f.states, state)
|
|
return f.record("scope_state")
|
|
}
|
|
func (f *fakeFinishStore) ReleaseInstanceMaintenance(context.Context, string, string, time.Time) error {
|
|
if err := f.releasedSentinel(); err != nil {
|
|
return err
|
|
}
|
|
return f.record("instance_release")
|
|
}
|
|
func (f *fakeFinishStore) ReleaseLibraryMaintenance(context.Context, string, string, time.Time) error {
|
|
if err := f.releasedSentinel(); err != nil {
|
|
return err
|
|
}
|
|
return f.record("scope_release")
|
|
}
|
|
func (f *fakeFinishStore) AdvanceLibraryJob(_ context.Context, _ string, state governance.LibraryJobState,
|
|
_, _, _, _, _ string, _ time.Time) error {
|
|
f.advanced = append(f.advanced, state)
|
|
return f.record("advance_" + string(state))
|
|
}
|
|
|
|
type fakeHealth struct{ err error }
|
|
|
|
func (h fakeHealth) VerifyRunning(context.Context) error { return h.err }
|
|
|
|
func boundary() successBoundary {
|
|
return successBoundary{Ready: true, Backup: "backup-1", Target: "snap-1", FinishedAt: time.Now().UTC()}
|
|
}
|
|
|
|
// 首次观察到成功时,两把维护锁必须已经释放,且终态写入排在最后。
|
|
func TestFinishLibrarySuccessReleasesEveryLockBeforeSucceeded(t *testing.T) {
|
|
store := &fakeFinishStore{}
|
|
if err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-1", boundary()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := []string{"scope_ingress", "instance_ingress", "scope_state", "scope_release",
|
|
"instance_state", "instance_release", "advance_succeeded"}
|
|
if strings.Join(store.calls, ",") != strings.Join(want, ",") {
|
|
t.Fatalf("call order = %v", store.calls)
|
|
}
|
|
if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobSucceeded {
|
|
t.Fatalf("advanced = %v", store.advanced)
|
|
}
|
|
}
|
|
|
|
// 目标服务/入口没恢复健康:不许放锁,不许报成功,作业落成失败并保持维护。
|
|
func TestFinishLibrarySuccessKeepsMaintenanceWhenTargetIsNotHealthy(t *testing.T) {
|
|
store := &fakeFinishStore{}
|
|
err := finishLibrarySuccess(context.Background(), store, fakeHealth{err: errors.New("app not healthy")},
|
|
"kgseal-weknora", "internal", "job-2", boundary())
|
|
if err == nil || !strings.Contains(err.Error(), "未恢复健康") {
|
|
t.Fatalf("err = %v", err)
|
|
}
|
|
for _, call := range store.calls {
|
|
if strings.HasSuffix(call, "_release") || call == "advance_succeeded" {
|
|
t.Fatalf("released or reported success despite unhealthy target: %v", store.calls)
|
|
}
|
|
}
|
|
if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed {
|
|
t.Fatalf("advanced = %v", store.advanced)
|
|
}
|
|
}
|
|
|
|
// 任一步释放失败:不报成功,作业失败,维护状态写 needs_restore(等人工/接管)。
|
|
func TestFinishLibrarySuccessStopsOnReleaseFailure(t *testing.T) {
|
|
for _, step := range []string{"scope_release", "instance_state", "instance_release"} {
|
|
t.Run(step, func(t *testing.T) {
|
|
store := &fakeFinishStore{failOn: step}
|
|
err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-3", boundary())
|
|
if err == nil {
|
|
t.Fatalf("%s failure was swallowed", step)
|
|
}
|
|
for _, call := range store.calls {
|
|
if call == "advance_succeeded" {
|
|
t.Fatalf("reported success after %s failed: %v", step, store.calls)
|
|
}
|
|
}
|
|
if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed {
|
|
t.Fatalf("advanced = %v", store.advanced)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// 终态写入失败:锁已释放,但不能声称成功;作业保持非终态等接管。
|
|
func TestFinishLibrarySuccessReportsTerminalWriteFailure(t *testing.T) {
|
|
store := &fakeFinishStore{failOn: "advance_succeeded"}
|
|
err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-4", boundary())
|
|
if err == nil || !strings.Contains(err.Error(), "等待接管") {
|
|
t.Fatalf("err = %v", err)
|
|
}
|
|
}
|
|
|
|
// 接管分类:阶段证据独立成立;查询失败=未知(不得写入);被顶替=不得覆盖;
|
|
// 台账自相矛盾(未顶替但不是当前生效 / 已顶替却是当前生效)=未知,fail closed。
|
|
func TestClassifyTakeoverSeparatesUnknownPendingSupersededAndFresh(t *testing.T) {
|
|
job := governance.LibraryJob{ID: "job-own", Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "snap-1"}
|
|
finalized := job
|
|
finalized.Stages = []governance.LibraryStage{{Stage: "finalize", State: "succeeded"}}
|
|
supersededAt := time.Now().UTC()
|
|
|
|
// source 是"这一行生效登记属于哪个作业":快照相等不代表本作业写过。
|
|
factsBy := func(source, current string, hasCurrent bool, snapshot string, hasSnapshot bool, superseded bool) libraryVersionFacts {
|
|
result := libraryVersionFacts{HasCurrent: hasCurrent, HasSnapshot: hasSnapshot}
|
|
if hasCurrent {
|
|
result.Current = governance.LibraryVersion{Kind: governance.LibraryVersionProductionEffective,
|
|
SnapshotID: current, SourceJobID: source}
|
|
}
|
|
if hasSnapshot {
|
|
result.Snapshot = governance.LibraryVersion{Kind: governance.LibraryVersionProductionEffective,
|
|
SnapshotID: snapshot, SourceJobID: source}
|
|
if superseded {
|
|
result.Snapshot.SupersededAt = &supersededAt
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
facts := func(current string, hasCurrent bool, snapshot string, hasSnapshot bool, superseded bool) libraryVersionFacts {
|
|
return factsBy(job.ID, current, hasCurrent, snapshot, hasSnapshot, superseded)
|
|
}
|
|
queryErr := libraryVersionFacts{Err: errors.New("version storage temporarily unavailable")}
|
|
|
|
cases := []struct {
|
|
name string
|
|
job governance.LibraryJob
|
|
facts libraryVersionFacts
|
|
want takeoverKind
|
|
}{
|
|
// 关键回归:查询失败 + 没有阶段证据 ⇒ 未知,不能当成"没有收尾证据"继续写。
|
|
{"query fails without stage evidence", job, queryErr, takeoverUnknown},
|
|
// 阶段证据先独立成立:查询失败不影响"只差收尾"的判定。
|
|
{"query fails but finalize stage succeeded", finalized, queryErr, takeoverPendingFinal},
|
|
{"effective version for this snapshot", job, facts("snap-1", true, "snap-1", true, false), takeoverPendingFinal},
|
|
// 当前生效的就是本快照,但没有本作业的登记行:归属仍是本作业才算只差收尾。
|
|
{"current is this snapshot without own row", job, facts("snap-1", true, "", false, false), takeoverPendingFinal},
|
|
// 实测缺陷(2026-10-29):同一个快照由**别的作业**登记成生效版本时,本作业只写完了
|
|
// database 阶段也被判成只差收尾 ⇒ 直接成功并改写版本归属。必须判 foreign。
|
|
{"same snapshot registered by another job", job,
|
|
factsBy("libsync-internal-other", "snap-1", true, "snap-1", true, false), takeoverForeignEffective},
|
|
{"current is this snapshot owned by another job", job,
|
|
factsBy("libsync-internal-other", "snap-1", true, "", false, false), takeoverForeignEffective},
|
|
// 本作业自己的 finalize 阶段成功是独立证据:即使版本归属被别人改写,也不算未知。
|
|
{"finalize stage succeeded even if ownership was rewritten", finalized,
|
|
factsBy("libsync-internal-other", "snap-1", true, "snap-1", true, false), takeoverPendingFinal},
|
|
{"finalize stage succeeded", finalized, libraryVersionFacts{}, takeoverPendingFinal},
|
|
{"this snapshot was superseded", job, facts("snap-9", true, "snap-1", true, true), takeoverSuperseded},
|
|
{"no evidence at all", job, libraryVersionFacts{}, takeoverFresh},
|
|
{"another snapshot effective but this one has no row", job, facts("snap-9", true, "", false, false), takeoverFresh},
|
|
{"empty target snapshot", governance.LibraryJob{Scope: "internal"}, libraryVersionFacts{}, takeoverFresh},
|
|
// 未顶替的登记行却不是当前生效版本:两个事实互相矛盾,不许挑一个继续写。
|
|
{"own row not superseded but current differs", job, facts("snap-9", true, "snap-1", true, false), takeoverUnknown},
|
|
// 已顶替却又是当前生效版本:同样是矛盾台账。
|
|
{"own row superseded but current is it", job, facts("snap-1", true, "snap-1", true, true), takeoverUnknown},
|
|
// 有登记行但读不到当前生效版本:无法确认它是不是生效版本。
|
|
{"own row without current", job, facts("", false, "snap-1", true, false), takeoverUnknown},
|
|
}
|
|
for _, test := range cases {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if got := classifyTakeover(test.job, test.facts, takeoverResumeFacts{}); got != test.want {
|
|
t.Fatalf("classify = %q, want %q", got, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// 部分释放后重试必须幂等:已由本作业释放的行不能再算失败。
|
|
func TestReleaseWindowLocksToleratesAlreadyReleasedRows(t *testing.T) {
|
|
store := &fakeFinishStore{}
|
|
if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1",
|
|
"verified", time.Now().UTC()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store = &fakeFinishStore{alreadyReleased: true}
|
|
if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1",
|
|
"verified", time.Now().UTC()); err != nil {
|
|
t.Fatalf("retry after a partial release must be idempotent: %v", err)
|
|
}
|
|
// 真正"不是本作业持有"的错误仍然要暴露。
|
|
store = &fakeFinishStore{failOn: "instance_release"}
|
|
if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1",
|
|
"verified", time.Now().UTC()); err == nil {
|
|
t.Fatal("a foreign owner error was swallowed")
|
|
}
|
|
}
|
|
|
|
func ptrTime(value time.Time) *time.Time { return &value }
|