fix(library): 预备份硬门禁 + 崩溃接管 + 五项入口验收
build / build (push) Failing after 17s

为什么:父审指出预备份引用持久化失败仍继续写目标、接管依赖启动扫描且漏掉租约未过期、
恢复失败被标成"无需恢复"、版本登记与作业状态分叉。

- BackupPrepared 返回 error,持久化失败即零写入;预备份同时登记 production_prebackup 版本。
- 接管改为按队列租约判定 + 运行期 30s 扫描 + 原子抢占 + 固定幂等键 + attempt 归零。
- 接管场景保留预备份、不再宣称"未写无需恢复"。
- 恢复/重放同一快照重新激活 production_effective(清 superseded_at)。
- finalize 续作只做校验+登记(finalize_only),不重写生产。
- 索引映射基线缺失时采用首次观察值并登记;索引校验加孤儿文档检查(按 knowledge_id)。

真实验收(候选 API :18089 + 真实 PG/ES/MinIO):
A0 首次发布、A1 撤销、B 按版本恢复、C 写阶段 SIGKILL 接管、
D 跨 kind 互斥与多 worker 单次执行、E1 登记失败、E2 finalize 续作、E3 预备份失败零写入。
This commit is contained in:
姚章浩
2026-10-28 19:11:37 +08:00
parent ca51cbe202
commit c72b71d78a
12 changed files with 529 additions and 118 deletions
+41 -4
View File
@@ -239,7 +239,13 @@ WHERE scope=$1 AND kind='production_effective' AND superseded_at IS NULL`, versi
if _, err := tx.Exec(ctx, `
INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id, evidence, created_by, created_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
ON CONFLICT (id) DO NOTHING`,
ON CONFLICT (id) DO UPDATE SET
-- 恢复/重放同一个快照时必须把它重新"激活":否则它还是被 supersede 的旧行,
-- 页面会显示"生产当前版本未知",而实际生产已经恢复到这个快照。
superseded_at = NULL,
source_job_id = EXCLUDED.source_job_id,
evidence = EXCLUDED.evidence,
created_at = EXCLUDED.created_at`,
version.ID, version.Scope, string(version.Kind), version.SnapshotID, version.VersionLabel,
version.SourceJobID, evidence, version.CreatedBy, version.CreatedAt.UTC()); err != nil {
return fmt.Errorf("record library version: %w", err)
@@ -250,10 +256,19 @@ ON CONFLICT (id) DO NOTHING`,
// InterruptedLibraryJobs 找出"业务作业 running、但队列里已没有活跃作业"的作业:
// worker 被杀/重启后它们需要被接管,而不是被当成失败丢掉已完成的预备份。
func (s *Store) InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error) {
// lease 过期才算中断:worker 刚被杀但 lease 未到期时,队列里还是 running,
// 直接判"没中断"会让作业永远悬着;所以这里按 lease_until 判断。
rows, err := s.db.Query(ctx, `
SELECT id FROM library_jobs lj
WHERE lj.state='running' AND lj.kind IN ('sync','restore')
AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.id=lj.queue_job_id AND j.state IN ('pending','running','retry'))
SELECT lj.id FROM library_jobs lj
LEFT JOIN jobs j ON j.id=lj.queue_job_id
WHERE lj.kind IN ('sync','restore')
AND (
-- 业务作业在跑、队列已经没了/停了/租约过期
(lj.state='running' AND (j.id IS NULL OR j.state IN ('failed','completed','cancelled','dead')
OR (j.state IN ('pending','running','retry') AND j.lease_until IS NOT NULL AND j.lease_until < now())))
-- 业务作业已被前一轮接管置为 pending,但队列作业的租约也没了(worker 又死了)
OR (lj.state='pending' AND j.state='running' AND j.lease_until IS NOT NULL AND j.lease_until < now())
)
ORDER BY lj.created_at`)
if err != nil {
return nil, fmt.Errorf("list interrupted library jobs: %w", err)
@@ -281,6 +296,28 @@ ORDER BY lj.created_at`)
return jobs, nil
}
// ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零,
// 否则 attempt<max_attempts 永远不成立,作业再也不会被任何 worker 领取。
func (s *Store) ResetQueueJobForResume(ctx context.Context, queueJobID string, now time.Time) error {
if _, err := s.db.Exec(ctx, `UPDATE jobs
SET state='pending', attempt=0, lease_owner=NULL, lease_until=NULL, available_at=$2, updated_at=$2
WHERE id=$1 AND state IN ('running','retry','failed','pending')`, queueJobID, now); err != nil {
return fmt.Errorf("reset queue job for resume: %w", err)
}
return nil
}
// ClaimLibraryJobForResume 用一次条件更新把中断作业抢过来:只有把 running 改成 pending
// 的那一个 worker 拿到 true,多个 worker 同时重启也只会有一个执行(fencing)。
func (s *Store) ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error) {
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET state='pending', updated_at=$2
WHERE id=$1 AND state IN ('running','pending')`, jobID, now)
if err != nil {
return false, fmt.Errorf("claim interrupted library job: %w", err)
}
return tag.RowsAffected() == 1, nil
}
// ReconcileLibraryJobs 收拾"业务作业还在飞、队列里已经没有对应作业"的孤儿:
// 它们会永久占用 (scope,kind) 唯一索引,让同一目标再也发不出作业。
// 只标失败并写明恢复责任,不猜结果、不重放。