package main import ( "context" "errors" "strings" "testing" "time" "knowledge-governance/internal/governance" postgresstore "knowledge-governance/internal/platform/postgres" ) type fakeFinishStore struct { calls []string failOn string alreadyReleased bool advanced []governance.LibraryJobState // states 记录所有维护状态写入(测试要区分 verified 与 needs_restore)。 states []string } // 模拟"这一行已由同一个作业释放过"(真实 Store 返回 ErrMaintenanceReleased)。 func (f *fakeFinishStore) releasedSentinel() error { if f.alreadyReleased { return postgresstore.ErrMaintenanceReleased } return nil } func (f *fakeFinishStore) record(name string) error { f.calls = append(f.calls, name) if f.failOn == name { return errors.New("injected " + name + " failure") } return nil } func (f *fakeFinishStore) SetInstanceMaintenanceIngress(context.Context, string, string, string, string, time.Time) error { if err := f.releasedSentinel(); err != nil { return err } return f.record("instance_ingress") } func (f *fakeFinishStore) SetLibraryMaintenanceIngress(context.Context, string, string, string, string, time.Time) error { if err := f.releasedSentinel(); err != nil { return err } return f.record("scope_ingress") } func (f *fakeFinishStore) UpdateInstanceMaintenanceState(_ context.Context, _, _, state, _ string, _ time.Time) error { if err := f.releasedSentinel(); err != nil { return err } f.states = append(f.states, state) return f.record("instance_state") } func (f *fakeFinishStore) UpdateLibraryMaintenanceState(_ context.Context, _, _, state, _ string, _ time.Time) error { if err := f.releasedSentinel(); err != nil { return err } f.states = append(f.states, state) return f.record("scope_state") } func (f *fakeFinishStore) ReleaseInstanceMaintenance(context.Context, string, string, time.Time) error { if err := f.releasedSentinel(); err != nil { return err } return f.record("instance_release") } func (f *fakeFinishStore) ReleaseLibraryMaintenance(context.Context, string, string, time.Time) error { if err := f.releasedSentinel(); err != nil { return err } return f.record("scope_release") } func (f *fakeFinishStore) AdvanceLibraryJob(_ context.Context, _ string, state governance.LibraryJobState, _, _, _, _, _ string, _ time.Time) error { f.advanced = append(f.advanced, state) return f.record("advance_" + string(state)) } type fakeHealth struct{ err error } func (h fakeHealth) VerifyRunning(context.Context) error { return h.err } func boundary() successBoundary { return successBoundary{Ready: true, Backup: "backup-1", Target: "snap-1", FinishedAt: time.Now().UTC()} } // 首次观察到成功时,两把维护锁必须已经释放,且终态写入排在最后。 func TestFinishLibrarySuccessReleasesEveryLockBeforeSucceeded(t *testing.T) { store := &fakeFinishStore{} if err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-1", boundary()); err != nil { t.Fatal(err) } want := []string{"scope_ingress", "instance_ingress", "scope_state", "scope_release", "instance_state", "instance_release", "advance_succeeded"} if strings.Join(store.calls, ",") != strings.Join(want, ",") { t.Fatalf("call order = %v", store.calls) } if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobSucceeded { t.Fatalf("advanced = %v", store.advanced) } } // 目标服务/入口没恢复健康:不许放锁,不许报成功,作业落成失败并保持维护。 func TestFinishLibrarySuccessKeepsMaintenanceWhenTargetIsNotHealthy(t *testing.T) { store := &fakeFinishStore{} err := finishLibrarySuccess(context.Background(), store, fakeHealth{err: errors.New("app not healthy")}, "kgseal-weknora", "internal", "job-2", boundary()) if err == nil || !strings.Contains(err.Error(), "未恢复健康") { t.Fatalf("err = %v", err) } for _, call := range store.calls { if strings.HasSuffix(call, "_release") || call == "advance_succeeded" { t.Fatalf("released or reported success despite unhealthy target: %v", store.calls) } } if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed { t.Fatalf("advanced = %v", store.advanced) } } // 任一步释放失败:不报成功,作业失败,维护状态写 needs_restore(等人工/接管)。 func TestFinishLibrarySuccessStopsOnReleaseFailure(t *testing.T) { for _, step := range []string{"scope_release", "instance_state", "instance_release"} { t.Run(step, func(t *testing.T) { store := &fakeFinishStore{failOn: step} err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-3", boundary()) if err == nil { t.Fatalf("%s failure was swallowed", step) } for _, call := range store.calls { if call == "advance_succeeded" { t.Fatalf("reported success after %s failed: %v", step, store.calls) } } if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed { t.Fatalf("advanced = %v", store.advanced) } }) } } // 终态写入失败:锁已释放,但不能声称成功;作业保持非终态等接管。 func TestFinishLibrarySuccessReportsTerminalWriteFailure(t *testing.T) { store := &fakeFinishStore{failOn: "advance_succeeded"} err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-4", boundary()) if err == nil || !strings.Contains(err.Error(), "等待接管") { t.Fatalf("err = %v", err) } } // 接管分类:阶段证据独立成立;查询失败=未知(不得写入);被顶替=不得覆盖; // 台账自相矛盾(未顶替但不是当前生效 / 已顶替却是当前生效)=未知,fail closed。 func TestClassifyTakeoverSeparatesUnknownPendingSupersededAndFresh(t *testing.T) { job := governance.LibraryJob{ID: "job-own", Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "snap-1"} finalized := job finalized.Stages = []governance.LibraryStage{{Stage: "finalize", State: "succeeded"}} supersededAt := time.Now().UTC() // source 是"这一行生效登记属于哪个作业":快照相等不代表本作业写过。 factsBy := func(source, current string, hasCurrent bool, snapshot string, hasSnapshot bool, superseded bool) libraryVersionFacts { result := libraryVersionFacts{HasCurrent: hasCurrent, HasSnapshot: hasSnapshot} if hasCurrent { result.Current = governance.LibraryVersion{Kind: governance.LibraryVersionProductionEffective, SnapshotID: current, SourceJobID: source} } if hasSnapshot { result.Snapshot = governance.LibraryVersion{Kind: governance.LibraryVersionProductionEffective, SnapshotID: snapshot, SourceJobID: source} if superseded { result.Snapshot.SupersededAt = &supersededAt } } return result } facts := func(current string, hasCurrent bool, snapshot string, hasSnapshot bool, superseded bool) libraryVersionFacts { return factsBy(job.ID, current, hasCurrent, snapshot, hasSnapshot, superseded) } queryErr := libraryVersionFacts{Err: errors.New("version storage temporarily unavailable")} cases := []struct { name string job governance.LibraryJob facts libraryVersionFacts want takeoverKind }{ // 关键回归:查询失败 + 没有阶段证据 ⇒ 未知,不能当成"没有收尾证据"继续写。 {"query fails without stage evidence", job, queryErr, takeoverUnknown}, // 阶段证据先独立成立:查询失败不影响"只差收尾"的判定。 {"query fails but finalize stage succeeded", finalized, queryErr, takeoverPendingFinal}, {"effective version for this snapshot", job, facts("snap-1", true, "snap-1", true, false), takeoverPendingFinal}, // 当前生效的就是本快照,但没有本作业的登记行:归属仍是本作业才算只差收尾。 {"current is this snapshot without own row", job, facts("snap-1", true, "", false, false), takeoverPendingFinal}, // 实测缺陷(2026-10-29):同一个快照由**别的作业**登记成生效版本时,本作业只写完了 // database 阶段也被判成只差收尾 ⇒ 直接成功并改写版本归属。必须判 foreign。 {"same snapshot registered by another job", job, factsBy("libsync-internal-other", "snap-1", true, "snap-1", true, false), takeoverForeignEffective}, {"current is this snapshot owned by another job", job, factsBy("libsync-internal-other", "snap-1", true, "", false, false), takeoverForeignEffective}, // 本作业自己的 finalize 阶段成功是独立证据:即使版本归属被别人改写,也不算未知。 {"finalize stage succeeded even if ownership was rewritten", finalized, factsBy("libsync-internal-other", "snap-1", true, "snap-1", true, false), takeoverPendingFinal}, {"finalize stage succeeded", finalized, libraryVersionFacts{}, takeoverPendingFinal}, {"this snapshot was superseded", job, facts("snap-9", true, "snap-1", true, true), takeoverSuperseded}, {"no evidence at all", job, libraryVersionFacts{}, takeoverFresh}, {"another snapshot effective but this one has no row", job, facts("snap-9", true, "", false, false), takeoverFresh}, {"empty target snapshot", governance.LibraryJob{Scope: "internal"}, libraryVersionFacts{}, takeoverFresh}, // 未顶替的登记行却不是当前生效版本:两个事实互相矛盾,不许挑一个继续写。 {"own row not superseded but current differs", job, facts("snap-9", true, "snap-1", true, false), takeoverUnknown}, // 已顶替却又是当前生效版本:同样是矛盾台账。 {"own row superseded but current is it", job, facts("snap-1", true, "snap-1", true, true), takeoverUnknown}, // 有登记行但读不到当前生效版本:无法确认它是不是生效版本。 {"own row without current", job, facts("", false, "snap-1", true, false), takeoverUnknown}, } for _, test := range cases { t.Run(test.name, func(t *testing.T) { if got := classifyTakeover(test.job, test.facts, takeoverResumeFacts{}); got != test.want { t.Fatalf("classify = %q, want %q", got, test.want) } }) } } // 部分释放后重试必须幂等:已由本作业释放的行不能再算失败。 func TestReleaseWindowLocksToleratesAlreadyReleasedRows(t *testing.T) { store := &fakeFinishStore{} if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1", "verified", time.Now().UTC()); err != nil { t.Fatal(err) } store = &fakeFinishStore{alreadyReleased: true} if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1", "verified", time.Now().UTC()); err != nil { t.Fatalf("retry after a partial release must be idempotent: %v", err) } // 真正"不是本作业持有"的错误仍然要暴露。 store = &fakeFinishStore{failOn: "instance_release"} if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1", "verified", time.Now().UTC()); err == nil { t.Fatal("a foreign owner error was swallowed") } } func ptrTime(value time.Time) *time.Time { return &value }