fix(worker): 接管判据绑作业身份,同快照别作业登记不算完成
2026-10-29T19:16Z 真实现场:作业只写完 database 阶段(resources/index/wiki/verify 仍 pending),worker 被杀、人工拉起后被判 succeeded,阶段写"已登记生效版本(仅收尾, 未重写生产)",并把生效版本归属改写成这个作业。 根因:classifyTakeover 只看"当前生效版本的目标快照是不是我的目标快照"。版本行主键 由 (kind, scope, 快照) 决定(LibraryVersionID),登记是 ON CONFLICT (id) DO UPDATE、 会改写 source_job_id —— 快照相等这一维里没有作业身份。 改法:effectiveOwnedBy(version, jobID, snapshotID) 一份实现,接管与基线共用; 归属别的作业 ⇒ 新的 takeoverForeignEffective,走 failLibraryTakeover:不写目标、 不释放维护、终态失败并指明预备份(目标可能半写,释放等于谎称已恢复)。 本作业自己的 finalize 阶段成功仍是独立证据,不被归属改写推翻。 验证(已跑): - go test ./cmd/worker -run TestClassifyTakeover → PASS(含 4 条新用例) - 负例:撤掉身份判定后同两条用例红(classify="pending_final", want "foreign_effective") - 真实 PG(一次性库 kg_takeover_foreign_test): TestTakeoverRefusesSameSnapshotRegisteredByAnotherJob PASS(1.41s), 负例同一用例红(library_takeover_integration_test.go:53) - 同一库 TestTakeoverClassificationUsesExactCurrentVersionBeyondHistoryWindow PASS - make verify:overall success(15 项必需全绿)
This commit is contained in:
@@ -40,7 +40,7 @@ func classifyBaselineVersions(current governance.LibraryVersion, hasCurrent bool
|
||||
if !hasCurrent {
|
||||
return baselineVersionState{}
|
||||
}
|
||||
if jobID != "" && snapshotID != "" && current.SourceJobID == jobID && current.SnapshotID == snapshotID {
|
||||
if effectiveOwnedBy(current, jobID, snapshotID) {
|
||||
return baselineVersionState{Own: true}
|
||||
}
|
||||
foreign := current
|
||||
|
||||
@@ -114,14 +114,30 @@ const (
|
||||
takeoverPendingFinal takeoverKind = "pending_final"
|
||||
// takeoverSuperseded:本快照曾经生效、后来被别的快照顶替,不得自动覆盖。
|
||||
takeoverSuperseded takeoverKind = "superseded"
|
||||
// takeoverForeignEffective:当前生效的确实是本作业的目标快照,但那一行登记属于
|
||||
// **别的作业**。快照相同不等于本作业写过:2026-10-29 的真实现场就是这样——
|
||||
// 本作业只写完了第一个写入阶段(database),resources/index/wiki/verify 仍是
|
||||
// pending,却因为"当前生效快照 == 我的目标快照"被当成只差收尾,直接登记成功,
|
||||
// 并把生效版本归属改写成自己(同一 (scope, 快照) 的版本行是 upsert)。
|
||||
takeoverForeignEffective takeoverKind = "foreign_effective"
|
||||
// takeoverUnknown:版本状态读不到且没有阶段证据,属"未知",不得写入。
|
||||
takeoverUnknown takeoverKind = "unknown"
|
||||
)
|
||||
|
||||
// effectiveOwnedBy 回答"这一行生效登记是不是本作业为这个快照登记的"。
|
||||
//
|
||||
// 快照相等的判定共用于接管与基线两条链路(同一含义只准有一份实现):这两个
|
||||
// 调用方都必须绑作业身份,否则"别的作业把同一个快照发布上线"会被误认成
|
||||
// "本作业已经完成"。
|
||||
func effectiveOwnedBy(version governance.LibraryVersion, jobID, snapshotID string) bool {
|
||||
return jobID != "" && snapshotID != "" && version.SourceJobID == jobID && version.SnapshotID == snapshotID
|
||||
}
|
||||
|
||||
// classifyTakeover 只按证据分类:
|
||||
// - finalize 阶段成功是独立证据(来自作业自己的阶段行),不因版本查询失败而丢弃;
|
||||
// - 本快照存在 released/被顶替的版本 ⇒ superseded;
|
||||
// - 本快照存在未被顶掉的生效版本 ⇒ pending_final;
|
||||
// - 本快照存在未被顶掉的生效版本,且归属本作业 ⇒ pending_final;归属别的作业 ⇒
|
||||
// foreign_effective(同快照不算本作业的完成证据);
|
||||
// - 查询失败且无阶段证据 ⇒ unknown(可重试,不当成没有证据)。
|
||||
//
|
||||
// 事实来自两个精确查询(LibraryCurrentVersion / LibraryVersionOfSnapshot),**不是**
|
||||
@@ -151,11 +167,17 @@ func classifyTakeover(job governance.LibraryJob, facts libraryVersionFacts) take
|
||||
if !facts.HasCurrent || facts.Current.SnapshotID != job.TargetSnapshotID {
|
||||
return takeoverUnknown
|
||||
}
|
||||
if !finalized && !effectiveOwnedBy(facts.Snapshot, job.ID, job.TargetSnapshotID) {
|
||||
return takeoverForeignEffective
|
||||
}
|
||||
return takeoverPendingFinal
|
||||
}
|
||||
// 本快照没有自己的生效登记,但当前生效的正是它(恢复/重放会改写作业归属):
|
||||
// 内容已经在生产上,只差收尾。
|
||||
// 本快照没有自己的生效登记,但当前生效的正是它:内容确实已经在生产上。
|
||||
// 归属必须也是本作业,否则只能说明别的作业发布过同一快照。
|
||||
if facts.HasCurrent && facts.Current.SnapshotID == job.TargetSnapshotID {
|
||||
if !finalized && !effectiveOwnedBy(facts.Current, job.ID, job.TargetSnapshotID) {
|
||||
return takeoverForeignEffective
|
||||
}
|
||||
return takeoverPendingFinal
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,26 +155,30 @@ func TestFinishLibrarySuccessReportsTerminalWriteFailure(t *testing.T) {
|
||||
// 接管分类:阶段证据独立成立;查询失败=未知(不得写入);被顶替=不得覆盖;
|
||||
// 台账自相矛盾(未顶替但不是当前生效 / 已顶替却是当前生效)=未知,fail closed。
|
||||
func TestClassifyTakeoverSeparatesUnknownPendingSupersededAndFresh(t *testing.T) {
|
||||
job := governance.LibraryJob{Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "snap-1"}
|
||||
job := governance.LibraryJob{ID: "job-own", Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "snap-1"}
|
||||
finalized := job
|
||||
finalized.Stages = []governance.LibraryStage{{Stage: "finalize", State: "succeeded"}}
|
||||
supersededAt := time.Now().UTC()
|
||||
|
||||
facts := func(current string, hasCurrent bool, snapshot string, hasSnapshot bool, superseded bool) libraryVersionFacts {
|
||||
// source 是"这一行生效登记属于哪个作业":快照相等不代表本作业写过。
|
||||
factsBy := func(source, current string, hasCurrent bool, snapshot string, hasSnapshot bool, superseded bool) libraryVersionFacts {
|
||||
result := libraryVersionFacts{HasCurrent: hasCurrent, HasSnapshot: hasSnapshot}
|
||||
if hasCurrent {
|
||||
result.Current = governance.LibraryVersion{Kind: governance.LibraryVersionProductionEffective,
|
||||
SnapshotID: current, SourceJobID: "job-cur"}
|
||||
SnapshotID: current, SourceJobID: source}
|
||||
}
|
||||
if hasSnapshot {
|
||||
result.Snapshot = governance.LibraryVersion{Kind: governance.LibraryVersionProductionEffective,
|
||||
SnapshotID: snapshot, SourceJobID: job.ID}
|
||||
SnapshotID: snapshot, SourceJobID: source}
|
||||
if superseded {
|
||||
result.Snapshot.SupersededAt = &supersededAt
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
facts := func(current string, hasCurrent bool, snapshot string, hasSnapshot bool, superseded bool) libraryVersionFacts {
|
||||
return factsBy(job.ID, current, hasCurrent, snapshot, hasSnapshot, superseded)
|
||||
}
|
||||
queryErr := libraryVersionFacts{Err: errors.New("version storage temporarily unavailable")}
|
||||
|
||||
cases := []struct {
|
||||
@@ -188,8 +192,17 @@ func TestClassifyTakeoverSeparatesUnknownPendingSupersededAndFresh(t *testing.T)
|
||||
// 阶段证据先独立成立:查询失败不影响"只差收尾"的判定。
|
||||
{"query fails but finalize stage succeeded", finalized, queryErr, takeoverPendingFinal},
|
||||
{"effective version for this snapshot", job, facts("snap-1", true, "snap-1", true, false), takeoverPendingFinal},
|
||||
// 当前生效的就是本快照,但没有本作业的登记行(恢复/重放会改写作业归属)。
|
||||
// 当前生效的就是本快照,但没有本作业的登记行:归属仍是本作业才算只差收尾。
|
||||
{"current is this snapshot without own row", job, facts("snap-1", true, "", false, false), takeoverPendingFinal},
|
||||
// 实测缺陷(2026-10-29):同一个快照由**别的作业**登记成生效版本时,本作业只写完了
|
||||
// database 阶段也被判成只差收尾 ⇒ 直接成功并改写版本归属。必须判 foreign。
|
||||
{"same snapshot registered by another job", job,
|
||||
factsBy("libsync-internal-other", "snap-1", true, "snap-1", true, false), takeoverForeignEffective},
|
||||
{"current is this snapshot owned by another job", job,
|
||||
factsBy("libsync-internal-other", "snap-1", true, "", false, false), takeoverForeignEffective},
|
||||
// 本作业自己的 finalize 阶段成功是独立证据:即使版本归属被别人改写,也不算未知。
|
||||
{"finalize stage succeeded even if ownership was rewritten", finalized,
|
||||
factsBy("libsync-internal-other", "snap-1", true, "snap-1", true, false), takeoverPendingFinal},
|
||||
{"finalize stage succeeded", finalized, libraryVersionFacts{}, takeoverPendingFinal},
|
||||
{"this snapshot was superseded", job, facts("snap-9", true, "snap-1", true, true), takeoverSuperseded},
|
||||
{"no evidence at all", job, libraryVersionFacts{}, takeoverFresh},
|
||||
|
||||
@@ -138,6 +138,17 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
case takeoverSuperseded:
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"本快照的生效版本已被别的快照顶替,拒绝用旧快照覆盖当前生产版本", "prebackup")
|
||||
case takeoverForeignEffective:
|
||||
// 同一个快照由别的作业发布过:那只能证明生产上有这份内容,不能证明本作业
|
||||
// 写完过、更不能证明它只差收尾。按同快照收尾会把只写了第一个阶段的作业
|
||||
// 标成成功,并改写版本归属(实测过的假完成)。
|
||||
owner := facts.Current.SourceJobID
|
||||
if facts.HasSnapshot {
|
||||
owner = facts.Snapshot.SourceJobID
|
||||
}
|
||||
return failLibraryTakeover(ctx, store, recorder, payload.Scope, payload.LibraryJobID, previous.SnapshotID,
|
||||
"接管时目标快照 "+shortSnapshotID(payload.TargetSnapshotID)+" 的生效版本由作业 "+owner+
|
||||
" 登记:本作业没有自己的完成证据,不按同快照收尾")
|
||||
case takeoverPendingFinal:
|
||||
// 上次可能死在"锁已放、终态没写"之间:只复核与登记,不重写生产。
|
||||
payload.FinalizeOnly = true
|
||||
@@ -581,6 +592,23 @@ func failLibraryJob(ctx context.Context, store *postgresstore.Store, recorder li
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
// failLibraryTakeover 是"拒绝接管"的失败出口:目标可能已被上一次尝试写过,
|
||||
// 不能像 failLibraryJob 那样写"无需恢复"。维护占用保持现状(held/acquired 就是
|
||||
// 安全状态),scope 行显式标 needs_restore,作业终态失败并指明可回滚的预备份。
|
||||
//
|
||||
// 为什么不在拒绝时释放维护:本作业没有自己的写入证据,目标是否半写未知;
|
||||
// 释放就等于对页面和下一次作业谎称"目标已恢复原状"。实例行也不动——它的释放
|
||||
// 属于"按预备份恢复并核验"那条路径,这里连实例身份都还没查到。
|
||||
func failLibraryTakeover(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder,
|
||||
scope, jobID, backupSnapshotID, message string) error {
|
||||
recorder.Stage(ctx, "prebackup", "failed", message)
|
||||
recovery := "本作业未继续写目标;上一次尝试可能已半写,须按预备份 " + backupSnapshotID + " 恢复并核验后再处理"
|
||||
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "prebackup", message, recovery,
|
||||
backupSnapshotID, "", time.Now().UTC())
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "needs_restore", message, time.Now().UTC())
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
func kindLabel(kind governance.LibraryJobKind) string {
|
||||
if kind == governance.LibraryJobRestore {
|
||||
return "恢复"
|
||||
|
||||
@@ -9,6 +9,65 @@ import (
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 同一个快照被别的作业登记成生效版本时,接管必须拒绝:快照相等不是本作业的完成证据。
|
||||
//
|
||||
// 这条只能在真实 PostgreSQL 上证明:版本行 id 由 (kind, scope, 快照) 决定,登记走
|
||||
// upsert 并会改写 source_job_id,所以"生产上有这个快照、但它不是本作业发的"是存储层
|
||||
// 才能造出的真实形状。内存夹具只能造出调用方相信的那个版本。
|
||||
func TestTakeoverRefusesSameSnapshotRegisteredByAnotherJob(t *testing.T) {
|
||||
store, _ := openLedgerStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
const scope = "internal"
|
||||
const shared = "snap-shared-identical"
|
||||
const otherJob = "libsync-internal-publisher"
|
||||
const thisJob = "libsync-internal-resumed"
|
||||
|
||||
record := func(source string, created time.Time) {
|
||||
t.Helper()
|
||||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||||
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, scope, shared),
|
||||
Scope: scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: shared,
|
||||
SourceJobID: source, CreatedBy: "worker",
|
||||
Evidence: map[string]any{"action": "sync", "index_mapping_sha256": "mapping-live"},
|
||||
CreatedAt: created,
|
||||
}); err != nil {
|
||||
t.Fatalf("record effective for %s: %v", source, err)
|
||||
}
|
||||
}
|
||||
record(otherJob, now.Add(-time.Hour))
|
||||
|
||||
// 本作业被杀在写入链路的第一阶段:resources/index/wiki/verify 都还没开始。
|
||||
job := governance.LibraryJob{ID: thisJob, Scope: scope, Kind: governance.LibraryJobSync,
|
||||
TargetSnapshotID: shared, SnapshotID: "snap-prebackup-shared",
|
||||
Stages: []governance.LibraryStage{{Stage: "database", State: "running"}}}
|
||||
|
||||
facts := loadLibraryVersionFacts(ctx, store, scope, shared)
|
||||
if facts.Err != nil {
|
||||
t.Fatalf("read facts: %v", facts.Err)
|
||||
}
|
||||
if !facts.HasCurrent || !facts.HasSnapshot || facts.Current.SourceJobID != otherJob {
|
||||
t.Fatalf("夹具失效:生效登记不属于别的作业:%+v", facts)
|
||||
}
|
||||
if got := classifyTakeover(job, facts); got != takeoverForeignEffective {
|
||||
t.Fatalf("同快照由别的作业登记时的接管分类 = %q, want %q(按同快照收尾 = 假完成)",
|
||||
got, takeoverForeignEffective)
|
||||
}
|
||||
|
||||
// 本作业自己登记过同一快照(upsert 把归属改写回来)⇒ 只差收尾。
|
||||
record(thisJob, now)
|
||||
facts = loadLibraryVersionFacts(ctx, store, scope, shared)
|
||||
if facts.Err != nil {
|
||||
t.Fatalf("read facts after own record: %v", facts.Err)
|
||||
}
|
||||
if facts.Current.SourceJobID != thisJob {
|
||||
t.Fatalf("upsert 没有把归属改成本作业:%+v", facts.Current)
|
||||
}
|
||||
if got := classifyTakeover(job, facts); got != takeoverPendingFinal {
|
||||
t.Fatalf("本作业登记的同一快照分类 = %q, want %q", got, takeoverPendingFinal)
|
||||
}
|
||||
}
|
||||
|
||||
// 接管/基线的版本判定必须走"当前生效版本"的精确查询:历史一长,窗口列表就看不到
|
||||
// 那唯一一行未顶替的生效版本,于是判定会退回"没有证据"——接管就会把已经生效的
|
||||
// 目标再写一遍(重写风险最高的一条)。这里用真实 PostgreSQL 把窗口挤满,证明判定不变。
|
||||
|
||||
@@ -232,6 +232,11 @@
|
||||
"package": "knowledge-governance/cmd/worker",
|
||||
"test": "TestTakeoverClassificationUsesExactCurrentVersionBeyondHistoryWindow",
|
||||
"reason": "必须在真实 PostgreSQL(TEST_DATABASE_URL 指向可销毁测试库)上跑:要证明的是『当前生效版本』不能按最近 N 条找——先灌 250 条历史把窗口挤满,再断言接管判定仍是 pending_final / superseded;假 store 不实现窗口语义,默认 CI 无库 ⇒ 跳过(补验路径:起一次性 PG 后设 TEST_DATABASE_URL 单跑这条)"
|
||||
},
|
||||
{
|
||||
"package": "knowledge-governance/cmd/worker",
|
||||
"test": "TestTakeoverRefusesSameSnapshotRegisteredByAnotherJob",
|
||||
"reason": "必须在真实 PostgreSQL(TEST_DATABASE_URL 指向可销毁测试库)上跑:要证明的是『同一个快照由别的作业登记成生效版本』不能当成本作业的完成证据——版本行 id 由 (kind, scope, 快照) 决定且写入是 upsert(会改写 source_job_id),这个形状只有存储层能造;默认 CI 无库 ⇒ 跳过,本机一次性库已实跑(含负例:撤掉身份判定后判成 pending_final)"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user