为什么:父审指出预备份引用持久化失败仍继续写目标、接管依赖启动扫描且漏掉租约未过期、 恢复失败被标成"无需恢复"、版本登记与作业状态分叉。 - BackupPrepared 返回 error,持久化失败即零写入;预备份同时登记 production_prebackup 版本。 - 接管改为按队列租约判定 + 运行期 30s 扫描 + 原子抢占 + 固定幂等键 + attempt 归零。 - 接管场景保留预备份、不再宣称"未写无需恢复"。 - 恢复/重放同一快照重新激活 production_effective(清 superseded_at)。 - finalize 续作只做校验+登记(finalize_only),不重写生产。 - 索引映射基线缺失时采用首次观察值并登记;索引校验加孤儿文档检查(按 knowledge_id)。 真实验收(候选 API :18089 + 真实 PG/ES/MinIO): A0 首次发布、A1 撤销、B 按版本恢复、C 写阶段 SIGKILL 接管、 D 跨 kind 互斥与多 worker 单次执行、E1 登记失败、E2 finalize 续作、E3 预备份失败零写入。
This commit is contained in:
@@ -239,7 +239,13 @@ WHERE scope=$1 AND kind='production_effective' AND superseded_at IS NULL`, versi
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id, evidence, created_by, created_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
-- 恢复/重放同一个快照时必须把它重新"激活":否则它还是被 supersede 的旧行,
|
||||
-- 页面会显示"生产当前版本未知",而实际生产已经恢复到这个快照。
|
||||
superseded_at = NULL,
|
||||
source_job_id = EXCLUDED.source_job_id,
|
||||
evidence = EXCLUDED.evidence,
|
||||
created_at = EXCLUDED.created_at`,
|
||||
version.ID, version.Scope, string(version.Kind), version.SnapshotID, version.VersionLabel,
|
||||
version.SourceJobID, evidence, version.CreatedBy, version.CreatedAt.UTC()); err != nil {
|
||||
return fmt.Errorf("record library version: %w", err)
|
||||
@@ -250,10 +256,19 @@ ON CONFLICT (id) DO NOTHING`,
|
||||
// InterruptedLibraryJobs 找出"业务作业 running、但队列里已没有活跃作业"的作业:
|
||||
// worker 被杀/重启后它们需要被接管,而不是被当成失败丢掉已完成的预备份。
|
||||
func (s *Store) InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error) {
|
||||
// lease 过期才算中断:worker 刚被杀但 lease 未到期时,队列里还是 running,
|
||||
// 直接判"没中断"会让作业永远悬着;所以这里按 lease_until 判断。
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id FROM library_jobs lj
|
||||
WHERE lj.state='running' AND lj.kind IN ('sync','restore')
|
||||
AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.id=lj.queue_job_id AND j.state IN ('pending','running','retry'))
|
||||
SELECT lj.id FROM library_jobs lj
|
||||
LEFT JOIN jobs j ON j.id=lj.queue_job_id
|
||||
WHERE lj.kind IN ('sync','restore')
|
||||
AND (
|
||||
-- 业务作业在跑、队列已经没了/停了/租约过期
|
||||
(lj.state='running' AND (j.id IS NULL OR j.state IN ('failed','completed','cancelled','dead')
|
||||
OR (j.state IN ('pending','running','retry') AND j.lease_until IS NOT NULL AND j.lease_until < now())))
|
||||
-- 业务作业已被前一轮接管置为 pending,但队列作业的租约也没了(worker 又死了)
|
||||
OR (lj.state='pending' AND j.state='running' AND j.lease_until IS NOT NULL AND j.lease_until < now())
|
||||
)
|
||||
ORDER BY lj.created_at`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list interrupted library jobs: %w", err)
|
||||
@@ -281,6 +296,28 @@ ORDER BY lj.created_at`)
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零,
|
||||
// 否则 attempt<max_attempts 永远不成立,作业再也不会被任何 worker 领取。
|
||||
func (s *Store) ResetQueueJobForResume(ctx context.Context, queueJobID string, now time.Time) error {
|
||||
if _, err := s.db.Exec(ctx, `UPDATE jobs
|
||||
SET state='pending', attempt=0, lease_owner=NULL, lease_until=NULL, available_at=$2, updated_at=$2
|
||||
WHERE id=$1 AND state IN ('running','retry','failed','pending')`, queueJobID, now); err != nil {
|
||||
return fmt.Errorf("reset queue job for resume: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClaimLibraryJobForResume 用一次条件更新把中断作业抢过来:只有把 running 改成 pending
|
||||
// 的那一个 worker 拿到 true,多个 worker 同时重启也只会有一个执行(fencing)。
|
||||
func (s *Store) ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error) {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET state='pending', updated_at=$2
|
||||
WHERE id=$1 AND state IN ('running','pending')`, jobID, now)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("claim interrupted library job: %w", err)
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
// ReconcileLibraryJobs 收拾"业务作业还在飞、队列里已经没有对应作业"的孤儿:
|
||||
// 它们会永久占用 (scope,kind) 唯一索引,让同一目标再也发不出作业。
|
||||
// 只标失败并写明恢复责任,不猜结果、不重放。
|
||||
|
||||
Reference in New Issue
Block a user