package main import ( "context" "errors" "strings" "testing" "time" "knowledge-governance/internal/governance" postgresstore "knowledge-governance/internal/platform/postgres" ) type fakeFinishStore struct { calls []string failOn string alreadyReleased bool advanced []governance.LibraryJobState // states 记录所有维护状态写入(测试要区分 verified 与 needs_restore)。 states []string } // 模拟"这一行已由同一个作业释放过"(真实 Store 返回 ErrMaintenanceReleased)。 func (f *fakeFinishStore) releasedSentinel() error { if f.alreadyReleased { return postgresstore.ErrMaintenanceReleased } return nil } func (f *fakeFinishStore) record(name string) error { f.calls = append(f.calls, name) if f.failOn == name { return errors.New("injected " + name + " failure") } return nil } func (f *fakeFinishStore) SetInstanceMaintenanceIngress(context.Context, string, string, string, string, time.Time) error { if err := f.releasedSentinel(); err != nil { return err } return f.record("instance_ingress") } func (f *fakeFinishStore) SetLibraryMaintenanceIngress(context.Context, string, string, string, string, time.Time) error { if err := f.releasedSentinel(); err != nil { return err } return f.record("scope_ingress") } func (f *fakeFinishStore) UpdateInstanceMaintenanceState(_ context.Context, _, _, state, _ string, _ time.Time) error { if err := f.releasedSentinel(); err != nil { return err } f.states = append(f.states, state) return f.record("instance_state") } func (f *fakeFinishStore) UpdateLibraryMaintenanceState(_ context.Context, _, _, state, _ string, _ time.Time) error { if err := f.releasedSentinel(); err != nil { return err } f.states = append(f.states, state) return f.record("scope_state") } func (f *fakeFinishStore) ReleaseInstanceMaintenance(context.Context, string, string, time.Time) error { if err := f.releasedSentinel(); err != nil { return err } return f.record("instance_release") } func (f *fakeFinishStore) ReleaseLibraryMaintenance(context.Context, string, string, time.Time) error { if err := f.releasedSentinel(); err != nil { return err } return f.record("scope_release") } func (f *fakeFinishStore) AdvanceLibraryJob(_ context.Context, _ string, state governance.LibraryJobState, _, _, _, _, _ string, _ time.Time) error { f.advanced = append(f.advanced, state) return f.record("advance_" + string(state)) } type fakeHealth struct{ err error } func (h fakeHealth) VerifyRunning(context.Context) error { return h.err } func boundary() successBoundary { return successBoundary{Ready: true, Backup: "backup-1", Target: "snap-1", FinishedAt: time.Now().UTC()} } // 首次观察到成功时,两把维护锁必须已经释放,且终态写入排在最后。 func TestFinishLibrarySuccessReleasesEveryLockBeforeSucceeded(t *testing.T) { store := &fakeFinishStore{} if err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-1", boundary()); err != nil { t.Fatal(err) } want := []string{"scope_ingress", "instance_ingress", "scope_state", "scope_release", "instance_state", "instance_release", "advance_succeeded"} if strings.Join(store.calls, ",") != strings.Join(want, ",") { t.Fatalf("call order = %v", store.calls) } if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobSucceeded { t.Fatalf("advanced = %v", store.advanced) } } // 目标服务/入口没恢复健康:不许放锁,不许报成功,作业落成失败并保持维护。 func TestFinishLibrarySuccessKeepsMaintenanceWhenTargetIsNotHealthy(t *testing.T) { store := &fakeFinishStore{} err := finishLibrarySuccess(context.Background(), store, fakeHealth{err: errors.New("app not healthy")}, "kgseal-weknora", "internal", "job-2", boundary()) if err == nil || !strings.Contains(err.Error(), "未恢复健康") { t.Fatalf("err = %v", err) } for _, call := range store.calls { if strings.HasSuffix(call, "_release") || call == "advance_succeeded" { t.Fatalf("released or reported success despite unhealthy target: %v", store.calls) } } if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed { t.Fatalf("advanced = %v", store.advanced) } } // 任一步释放失败:不报成功,作业失败,维护状态写 needs_restore(等人工/接管)。 func TestFinishLibrarySuccessStopsOnReleaseFailure(t *testing.T) { for _, step := range []string{"scope_release", "instance_state", "instance_release"} { t.Run(step, func(t *testing.T) { store := &fakeFinishStore{failOn: step} err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-3", boundary()) if err == nil { t.Fatalf("%s failure was swallowed", step) } for _, call := range store.calls { if call == "advance_succeeded" { t.Fatalf("reported success after %s failed: %v", step, store.calls) } } if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed { t.Fatalf("advanced = %v", store.advanced) } }) } } // 终态写入失败:锁已释放,但不能声称成功;作业保持非终态等接管。 func TestFinishLibrarySuccessReportsTerminalWriteFailure(t *testing.T) { store := &fakeFinishStore{failOn: "advance_succeeded"} err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-4", boundary()) if err == nil || !strings.Contains(err.Error(), "等待接管") { t.Fatalf("err = %v", err) } } // 接管分类:阶段证据独立成立;查询失败=未知(不得写入);被顶替=不得覆盖。 func TestClassifyTakeoverSeparatesUnknownPendingSupersededAndFresh(t *testing.T) { job := governance.LibraryJob{Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "snap-1"} finalized := job finalized.Stages = []governance.LibraryStage{{Stage: "finalize", State: "succeeded"}} effective := []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-1"}} supersededVersion := []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-1", SupersededAt: ptrTime(time.Now())}} unrelated := []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-9"}} queryErr := errors.New("version storage temporarily unavailable") cases := []struct { name string job governance.LibraryJob versions []governance.LibraryVersion err error want takeoverKind }{ // 关键回归:查询失败 + 没有阶段证据 ⇒ 未知,不能当成"没有收尾证据"继续写。 {"query fails without stage evidence", job, nil, queryErr, takeoverUnknown}, // 阶段证据先独立成立:查询失败不影响"只差收尾"的判定。 {"query fails but finalize stage succeeded", finalized, nil, queryErr, takeoverPendingFinal}, {"effective version for this snapshot", job, effective, nil, takeoverPendingFinal}, {"finalize stage succeeded", finalized, nil, nil, takeoverPendingFinal}, {"this snapshot was superseded", job, supersededVersion, nil, takeoverSuperseded}, {"no evidence at all", job, nil, nil, takeoverFresh}, {"another snapshot effective but this one has no row", job, unrelated, nil, takeoverFresh}, {"empty target snapshot", governance.LibraryJob{Scope: "internal"}, nil, nil, takeoverFresh}, } for _, test := range cases { t.Run(test.name, func(t *testing.T) { if got := classifyTakeover(test.job, test.versions, test.err); got != test.want { t.Fatalf("classify = %q, want %q", got, test.want) } }) } } // 部分释放后重试必须幂等:已由本作业释放的行不能再算失败。 func TestReleaseWindowLocksToleratesAlreadyReleasedRows(t *testing.T) { store := &fakeFinishStore{} if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1", "verified", time.Now().UTC()); err != nil { t.Fatal(err) } store = &fakeFinishStore{alreadyReleased: true} if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1", "verified", time.Now().UTC()); err != nil { t.Fatalf("retry after a partial release must be idempotent: %v", err) } // 真正"不是本作业持有"的错误仍然要暴露。 store = &fakeFinishStore{failOn: "instance_release"} if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1", "verified", time.Now().UTC()); err == nil { t.Fatal("a foreign owner error was swallowed") } } func ptrTime(value time.Time) *time.Time { return &value }