Files
knowledge-governance/cmd/worker/library_finish_test.go
T
姚章浩 24d2546cbb
build / build (push) Successful in 1m28s
fix(library): 接管不再把版本查询失败当成没有收尾证据
resumed 分支里 LibraryVersions 的错误被忽略,FinalizeOnly 保持 false 后继续完整
apply:对"已登记版本并释放窗口、只差终态"的作业,这等于把同一份内容重写一遍目标,
与"最终只复核"的承诺相反。

现在接管先分类:finalize 阶段成功作为独立证据(不受版本查询影响)→ 只收尾;查询
失败且无阶段证据 → 未知,零写入、可重试;本快照被别的快照顶替 → 拒绝覆盖;其余才
按正常流程。

同时修释放幂等:Update*/Release* 对"已由本作业释放"的行返回 ErrMaintenanceReleased,
收尾重试视为已完成(部分释放后不会永久卡死),别的作业仍报未持有。真 PG 负例验证
两种情形。
2026-10-29 14:24:12 +08:00

210 lines
8.3 KiB
Go

package main
import (
"context"
"errors"
"strings"
"testing"
"time"
"knowledge-governance/internal/governance"
postgresstore "knowledge-governance/internal/platform/postgres"
)
type fakeFinishStore struct {
calls []string
failOn string
alreadyReleased bool
advanced []governance.LibraryJobState
}
// 模拟"这一行已由同一个作业释放过"(真实 Store 返回 ErrMaintenanceReleased)。
func (f *fakeFinishStore) releasedSentinel() error {
if f.alreadyReleased {
return postgresstore.ErrMaintenanceReleased
}
return nil
}
func (f *fakeFinishStore) record(name string) error {
f.calls = append(f.calls, name)
if f.failOn == name {
return errors.New("injected " + name + " failure")
}
return nil
}
func (f *fakeFinishStore) SetInstanceMaintenanceIngress(context.Context, string, string, string, string, time.Time) error {
if err := f.releasedSentinel(); err != nil {
return err
}
return f.record("instance_ingress")
}
func (f *fakeFinishStore) SetLibraryMaintenanceIngress(context.Context, string, string, string, string, time.Time) error {
if err := f.releasedSentinel(); err != nil {
return err
}
return f.record("scope_ingress")
}
func (f *fakeFinishStore) UpdateInstanceMaintenanceState(context.Context, string, string, string, string, time.Time) error {
if err := f.releasedSentinel(); err != nil {
return err
}
return f.record("instance_state")
}
func (f *fakeFinishStore) UpdateLibraryMaintenanceState(context.Context, string, string, string, string, time.Time) error {
if err := f.releasedSentinel(); err != nil {
return err
}
return f.record("scope_state")
}
func (f *fakeFinishStore) ReleaseInstanceMaintenance(context.Context, string, string, time.Time) error {
if err := f.releasedSentinel(); err != nil {
return err
}
return f.record("instance_release")
}
func (f *fakeFinishStore) ReleaseLibraryMaintenance(context.Context, string, string, time.Time) error {
if err := f.releasedSentinel(); err != nil {
return err
}
return f.record("scope_release")
}
func (f *fakeFinishStore) AdvanceLibraryJob(_ context.Context, _ string, state governance.LibraryJobState,
_, _, _, _, _ string, _ time.Time) error {
f.advanced = append(f.advanced, state)
return f.record("advance_" + string(state))
}
type fakeHealth struct{ err error }
func (h fakeHealth) VerifyRunning(context.Context) error { return h.err }
func boundary() successBoundary {
return successBoundary{Ready: true, Backup: "backup-1", Target: "snap-1", FinishedAt: time.Now().UTC()}
}
// 首次观察到成功时,两把维护锁必须已经释放,且终态写入排在最后。
func TestFinishLibrarySuccessReleasesEveryLockBeforeSucceeded(t *testing.T) {
store := &fakeFinishStore{}
if err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-1", boundary()); err != nil {
t.Fatal(err)
}
want := []string{"scope_ingress", "instance_ingress", "scope_state", "scope_release",
"instance_state", "instance_release", "advance_succeeded"}
if strings.Join(store.calls, ",") != strings.Join(want, ",") {
t.Fatalf("call order = %v", store.calls)
}
if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobSucceeded {
t.Fatalf("advanced = %v", store.advanced)
}
}
// 目标服务/入口没恢复健康:不许放锁,不许报成功,作业落成失败并保持维护。
func TestFinishLibrarySuccessKeepsMaintenanceWhenTargetIsNotHealthy(t *testing.T) {
store := &fakeFinishStore{}
err := finishLibrarySuccess(context.Background(), store, fakeHealth{err: errors.New("app not healthy")},
"kgseal-weknora", "internal", "job-2", boundary())
if err == nil || !strings.Contains(err.Error(), "未恢复健康") {
t.Fatalf("err = %v", err)
}
for _, call := range store.calls {
if strings.HasSuffix(call, "_release") || call == "advance_succeeded" {
t.Fatalf("released or reported success despite unhealthy target: %v", store.calls)
}
}
if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed {
t.Fatalf("advanced = %v", store.advanced)
}
}
// 任一步释放失败:不报成功,作业失败,维护状态写 needs_restore(等人工/接管)。
func TestFinishLibrarySuccessStopsOnReleaseFailure(t *testing.T) {
for _, step := range []string{"scope_release", "instance_state", "instance_release"} {
t.Run(step, func(t *testing.T) {
store := &fakeFinishStore{failOn: step}
err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-3", boundary())
if err == nil {
t.Fatalf("%s failure was swallowed", step)
}
for _, call := range store.calls {
if call == "advance_succeeded" {
t.Fatalf("reported success after %s failed: %v", step, store.calls)
}
}
if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed {
t.Fatalf("advanced = %v", store.advanced)
}
})
}
}
// 终态写入失败:锁已释放,但不能声称成功;作业保持非终态等接管。
func TestFinishLibrarySuccessReportsTerminalWriteFailure(t *testing.T) {
store := &fakeFinishStore{failOn: "advance_succeeded"}
err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-4", boundary())
if err == nil || !strings.Contains(err.Error(), "等待接管") {
t.Fatalf("err = %v", err)
}
}
// 接管分类:阶段证据独立成立;查询失败=未知(不得写入);被顶替=不得覆盖。
func TestClassifyTakeoverSeparatesUnknownPendingSupersededAndFresh(t *testing.T) {
job := governance.LibraryJob{Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "snap-1"}
finalized := job
finalized.Stages = []governance.LibraryStage{{Stage: "finalize", State: "succeeded"}}
effective := []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-1"}}
supersededVersion := []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionEffective,
SnapshotID: "snap-1", SupersededAt: ptrTime(time.Now())}}
unrelated := []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-9"}}
queryErr := errors.New("version storage temporarily unavailable")
cases := []struct {
name string
job governance.LibraryJob
versions []governance.LibraryVersion
err error
want takeoverKind
}{
// 关键回归:查询失败 + 没有阶段证据 ⇒ 未知,不能当成"没有收尾证据"继续写。
{"query fails without stage evidence", job, nil, queryErr, takeoverUnknown},
// 阶段证据先独立成立:查询失败不影响"只差收尾"的判定。
{"query fails but finalize stage succeeded", finalized, nil, queryErr, takeoverPendingFinal},
{"effective version for this snapshot", job, effective, nil, takeoverPendingFinal},
{"finalize stage succeeded", finalized, nil, nil, takeoverPendingFinal},
{"this snapshot was superseded", job, supersededVersion, nil, takeoverSuperseded},
{"no evidence at all", job, nil, nil, takeoverFresh},
{"another snapshot effective but this one has no row", job, unrelated, nil, takeoverFresh},
{"empty target snapshot", governance.LibraryJob{Scope: "internal"}, nil, nil, takeoverFresh},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
if got := classifyTakeover(test.job, test.versions, test.err); got != test.want {
t.Fatalf("classify = %q, want %q", got, test.want)
}
})
}
}
// 部分释放后重试必须幂等:已由本作业释放的行不能再算失败。
func TestReleaseWindowLocksToleratesAlreadyReleasedRows(t *testing.T) {
store := &fakeFinishStore{}
if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1",
"verified", time.Now().UTC()); err != nil {
t.Fatal(err)
}
store = &fakeFinishStore{alreadyReleased: true}
if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1",
"verified", time.Now().UTC()); err != nil {
t.Fatalf("retry after a partial release must be idempotent: %v", err)
}
// 真正"不是本作业持有"的错误仍然要暴露。
store = &fakeFinishStore{failOn: "instance_release"}
if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1",
"verified", time.Now().UTC()); err == nil {
t.Fatal("a foreign owner error was swallowed")
}
}
func ptrTime(value time.Time) *time.Time { return &value }