Files
knowledge-governance/cmd/worker/library_takeover_integration_test.go
T
姚章浩 780ab988c4
build / build (push) Successful in 1m20s
fix(worker): 历史快照恢复接管按作业身份与新授权判定
真实现场 2026-10-30T00:19Z:新恢复作业恢复历史快照 fc595511(已被 0d982f76
顶替),写入 database 后被 SIGKILL;重启接管时 classifyTakeover 只要看到
该快照有 SupersededAt 就判 superseded,没有判这行历史版本属于哪个作业,
把历史那次发布被顶替当成这次作业过期,作业 failed/prebackup,而恢复文案
又写"无需恢复",目标半写且维护锁未释放。

修法:被顶替分支先分清归属——本作业 finalize 成功或版本行属于本作业 ⇒ 仍
拒绝;历史行属于别的作业时,仅当本次是 restore、授权签发于顶替之后、scope
与 instance 两把锁仍由本作业独占且指纹一致、预备份登记属于本作业且清单字节
仍等于摘要,才按 fresh 继续重跑整条链路。任一证据缺失/读不到仍拒绝。

同时把续作早退统一走 failLibraryJobOrTakeover:有持久化预备份时保留
needs_restore 并点名 backup,不再写"无需恢复"。

验证:
- 先写失败回归(历史行+新授权 ⇒ 期望 fresh,实得 superseded),修后绿;
- 真实 PostgreSQL 正负例:授权撤销/实例锁释放/预备份归属被改/清单字节被改
  均回到 superseded;续作早退保留备份与 needs_restore;
- 本地 make verify 16 项必需检查全绿(identity 本地档案跳过)。
2026-10-30 08:51:07 +08:00

230 lines
10 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"fmt"
"strings"
"testing"
"time"
"knowledge-governance/internal/governance"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// 同一个快照被别的作业登记成生效版本时,接管必须拒绝:快照相等不是本作业的完成证据。
//
// 这条只能在真实 PostgreSQL 上证明:版本行 id 由 (kind, scope, 快照) 决定,登记走
// upsert 并会改写 source_job_id,所以"生产上有这个快照、但它不是本作业发的"是存储层
// 才能造出的真实形状。内存夹具只能造出调用方相信的那个版本。
func TestTakeoverRefusesSameSnapshotRegisteredByAnotherJob(t *testing.T) {
store, _ := openLedgerStore(t)
ctx := context.Background()
now := time.Now().UTC()
const scope = "internal"
const shared = "snap-shared-identical"
const otherJob = "libsync-internal-publisher"
const thisJob = "libsync-internal-resumed"
record := func(source string, created time.Time) {
t.Helper()
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, scope, shared),
Scope: scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: shared,
SourceJobID: source, CreatedBy: "worker",
Evidence: map[string]any{"action": "sync", "index_mapping_sha256": "mapping-live"},
CreatedAt: created,
}); err != nil {
t.Fatalf("record effective for %s: %v", source, err)
}
}
record(otherJob, now.Add(-time.Hour))
// 本作业被杀在写入链路的第一阶段:resources/index/wiki/verify 都还没开始。
job := governance.LibraryJob{ID: thisJob, Scope: scope, Kind: governance.LibraryJobSync,
TargetSnapshotID: shared, SnapshotID: "snap-prebackup-shared",
Stages: []governance.LibraryStage{{Stage: "database", State: "running"}}}
facts := loadLibraryVersionFacts(ctx, store, scope, shared)
if facts.Err != nil {
t.Fatalf("read facts: %v", facts.Err)
}
if !facts.HasCurrent || !facts.HasSnapshot || facts.Current.SourceJobID != otherJob {
t.Fatalf("夹具失效:生效登记不属于别的作业:%+v", facts)
}
if got := classifyTakeover(job, facts, takeoverResumeFacts{}); got != takeoverForeignEffective {
t.Fatalf("同快照由别的作业登记时的接管分类 = %q, want %q(按同快照收尾 = 假完成)",
got, takeoverForeignEffective)
}
// 本作业自己登记过同一快照(upsert 把归属改写回来)⇒ 只差收尾。
record(thisJob, now)
facts = loadLibraryVersionFacts(ctx, store, scope, shared)
if facts.Err != nil {
t.Fatalf("read facts after own record: %v", facts.Err)
}
if facts.Current.SourceJobID != thisJob {
t.Fatalf("upsert 没有把归属改成本作业:%+v", facts.Current)
}
if got := classifyTakeover(job, facts, takeoverResumeFacts{}); got != takeoverPendingFinal {
t.Fatalf("本作业登记的同一快照分类 = %q, want %q", got, takeoverPendingFinal)
}
}
// "拒绝接管"的落库效果:作业终态失败、留预备份基线、**维护不释放**。
//
// 这条只能对真实 PostgreSQL 断言:三件事分别落在 library_jobs(state/recovery/
// snapshot_id)、library_job_stages(prebackup failed)与 library_maintenance
// (released_at 仍为 NULL、state=needs_restore)。假 store 只能证明"调用了函数"。
func TestFailLibraryTakeoverKeepsMaintenanceAndNamesBackup(t *testing.T) {
store, pool := openLedgerStore(t)
ctx := context.Background()
now := time.Now().UTC()
const scope = "internal"
const jobID = "libsync-internal-foreign-refused"
const backup = "snap-prebackup-refused"
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{
ID: jobID, Scope: scope, Kind: governance.LibraryJobSync, TargetSnapshotID: "snap-foreign",
RequestDigest: "digest-foreign", IdempotencyKey: "library.sync:foreign",
}, governance.SyncStages(), now); err != nil {
t.Fatalf("create job: %v", err)
}
if _, err := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: scope, OwnerJobID: jobID, TargetIdentity: "test-target", Level: "instance",
TargetFingerprint: "fingerprint", Services: []string{"app"}, IngressState: "closed",
State: "acquired", OriginalServiceState: "running", Note: "test",
}, now); err != nil {
t.Fatalf("acquire maintenance: %v", err)
}
recorder := libraryStageRecorder{store: store, jobID: jobID, scope: scope}
if err := failLibraryTakeover(ctx, store, recorder, scope, jobID, backup,
"接管时目标快照的生效版本由作业 other 登记"); err == nil {
t.Fatal("拒绝接管必须返回错误(不能安静地成功)")
}
job, err := store.LibraryJob(ctx, jobID)
if err != nil {
t.Fatalf("read job: %v", err)
}
if job.State != governance.LibraryJobFailed {
t.Fatalf("作业状态 = %s, want failed", job.State)
}
if job.SnapshotID != backup {
t.Fatalf("作业留存的预备份 = %q, want %q", job.SnapshotID, backup)
}
if !strings.Contains(job.Recovery, backup) {
t.Fatalf("恢复指引里没有预备份基线:%q", job.Recovery)
}
if strings.Contains(job.Recovery, "无需恢复") {
t.Fatalf("拒绝接管时不许写\"无需恢复\":%q", job.Recovery)
}
var released *time.Time
var state string
if err := pool.QueryRow(ctx,
`SELECT released_at, state FROM library_maintenance WHERE scope=$1`, scope).Scan(&released, &state); err != nil {
t.Fatalf("read maintenance: %v", err)
}
if released != nil {
t.Fatalf("维护被释放了(released_at=%s):目标可能半写,释放等于谎称已恢复", released)
}
if state != "needs_restore" {
t.Fatalf("维护状态 = %q, want needs_restore", state)
}
}
// 接管/基线的版本判定必须走"当前生效版本"的精确查询:历史一长,窗口列表就看不到
// 那唯一一行未顶替的生效版本,于是判定会退回"没有证据"——接管就会把已经生效的
// 目标再写一遍(重写风险最高的一条)。这里用真实 PostgreSQL 把窗口挤满,证明判定不变。
func TestTakeoverClassificationUsesExactCurrentVersionBeyondHistoryWindow(t *testing.T) {
store, _ := openLedgerStore(t)
ctx := context.Background()
now := time.Now().UTC()
const scope = "internal"
const targetSnapshot = "snap-effective"
const jobID = "libsync-internal-takeover"
job := governance.LibraryJob{ID: jobID, Scope: scope, Kind: governance.LibraryJobSync,
TargetSnapshotID: targetSnapshot, SnapshotID: "snap-prebackup"}
recordEffective := func(sourceJob, snapshot string, created time.Time) {
t.Helper()
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, scope, snapshot),
Scope: scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: snapshot,
SourceJobID: sourceJob, CreatedBy: "worker",
Evidence: map[string]any{"action": "sync", "index_mapping_sha256": "mapping-live"},
CreatedAt: created,
}); err != nil {
t.Fatalf("record effective %s: %v", snapshot, err)
}
}
// 生效版本登记在 48 小时前,然后 250 条更新的历史记录把它挤出任何窗口。
recordEffective(jobID, targetSnapshot, now.Add(-48*time.Hour))
seedVersionHistory(t, ctx, store, scope, now.Add(-time.Hour), 250)
window, err := store.LibraryVersions(ctx, scope, 100)
if err != nil {
t.Fatalf("list versions: %v", err)
}
if len(window) != 100 {
t.Fatalf("窗口条数 = %d, want 100", len(window))
}
if containsSnapshot(window, targetSnapshot) {
t.Fatal("窗口里仍能看到生效版本:本用例证明不了窗口之外仍可判定(夹具失效)")
}
// 本作业的目标快照就是当前生效版本 ⇒ 只差收尾,不许重写。
facts := loadLibraryVersionFacts(ctx, store, scope, targetSnapshot)
if facts.Err != nil {
t.Fatalf("窗口之外必须读得到生效事实:%v", facts.Err)
}
if !facts.HasCurrent || facts.Current.SnapshotID != targetSnapshot {
t.Fatalf("当前生效版本读取错误:%+v", facts)
}
if got := classifyTakeover(job, facts, takeoverResumeFacts{}); got != takeoverPendingFinal {
t.Fatalf("取手接管分类 = %q, want %q(窗口一满就重写 = 本缺陷)", got, takeoverPendingFinal)
}
// 索引映射基线也走同一条精确查询:窗口之外仍要拿到生产实际用的映射。
if got := knownIndexMapping(ctx, store, scope); got != "mapping-live" {
t.Fatalf("索引映射基线 = %q, want mapping-live", got)
}
// 本快照被更新的快照顶替:接管必须拒绝用旧快照覆盖,而不是重写。
recordEffective("libsync-internal-newer", "snap-newer", now)
facts = loadLibraryVersionFacts(ctx, store, scope, targetSnapshot)
if facts.Err != nil {
t.Fatalf("顶替后仍必须读得到事实:%v", facts.Err)
}
if got := classifyTakeover(job, facts, takeoverResumeFacts{}); got != takeoverSuperseded {
t.Fatalf("被顶替后的接管分类 = %q, want %q", got, takeoverSuperseded)
}
if facts.Current.SnapshotID != "snap-newer" {
t.Fatalf("当前生效版本 = %s, want snap-newer", facts.Current.SnapshotID)
}
// 一个从没生效过的快照:才是 fresh(正常流程 + 预检)。
unknown := governance.LibraryJob{ID: "libsync-internal-fresh", Scope: scope,
Kind: governance.LibraryJobSync, TargetSnapshotID: "snap-never", SnapshotID: "snap-prebackup-2"}
if got := classifyTakeover(unknown, loadLibraryVersionFacts(ctx, store, scope, "snap-never"), takeoverResumeFacts{}); got != takeoverFresh {
t.Fatalf("未生效过的快照分类 = %q, want %q", got, takeoverFresh)
}
// 基线侧同一个事实:别人的生效版本必须挡住建基线;本作业登记的同一快照只收尾。
current, hasCurrent, err := store.LibraryCurrentVersion(ctx, scope)
if err != nil || !hasCurrent {
t.Fatalf("当前生效版本读取失败:found=%v err=%v", hasCurrent, err)
}
if state := classifyBaselineVersions(current, hasCurrent, "libbase-internal-x", "snap-newer"); state.Own || state.Foreign == nil {
t.Fatalf("别人的生效版本必须挡住基线:%+v", state)
}
recordEffective("libbase-internal-y", "snap-baseline", now)
current, hasCurrent, err = store.LibraryCurrentVersion(ctx, scope)
if err != nil || !hasCurrent {
t.Fatalf("再读当前生效版本失败:found=%v err=%v", hasCurrent, err)
}
if state := classifyBaselineVersions(current, hasCurrent, "libbase-internal-y", "snap-baseline"); !state.Own {
t.Fatalf("本作业同一快照必须判成只收尾:%+v (%s)", state, fmt.Sprint(current))
}
}