R1 维护/发布守卫硬编码 j.id=$1:维护类入口的 $1 是 scope/instance,作业号在 $2, 导致 update/release/ingress 六个入口正常路径全部匹配不到作业。改为按被更新行自己的 owner_job_id 生成条件,并同时要求该代次在 library_attempts 里仍活跃。 R2 认领只改 jobs.attempt_epoch 不改 attempts.epoch,第二次接管即错位;唯一索引 (job_id) 与"每次接管新 attempt 且历史不可变"矛盾。改为同作业多代次:锁作业行与 operation 行、同 owner 幂等续租、活跃未过期者让路、过期/本作业旧代次标 abandoned、 新代次 epoch=max+1,并清同 operation 其它作业行租约。迟到 write/renew/finish 一律 ErrLibraryEpochStale(finish 补了"只能在活跃态进来一次")。 R3 续作不再新建 -resume 作业:PrepareLibraryJobResume 让原业务作业回 pending 并显式 结束上一代,维护窗口/journal/预备份/写集合仍按原作业 id 归属,代次由 worker 起手递增。 R4 执行域准入改为注册 handler 前读事实:目标实况(KB/租户/schema/索引)、旧 release 路由绑定的 KB、未释放维护窗口持有者;未知或重叠即拒绝执行(不做维护动作、不停既有服务)。 同时自纠 5 项设计偏离:current 次序键取意图创建时间(旧意图迟到事实被拒)、首次锚点 改事务 FOR UPDATE + 回读且 NULL≠0、版本事件冲突只做加法且 xmax=0 才算新事实、 恢复作为新 kind=recover 意图推进 current、隔离测试加测试库护栏并清理残留 schema。 测试:真实 PG(kg-pgtest-writeset,一用例一 schema)新增 8 条集成用例覆盖上述正负例; go test ./... 全绿(50 包)、go vet/build rc=0、web tsc + vitest 485 通过、make verify 16 项通过。
This commit is contained in:
@@ -354,6 +354,9 @@ type libraryJobStore interface {
|
||||
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
|
||||
AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error
|
||||
CancelLibraryJob(ctx context.Context, jobID, reason string, now time.Time) error
|
||||
// 续作走**原业务作业**(维护窗口/journal/预备份都按它落盘),只新增执行代次。
|
||||
PrepareLibraryJobResume(ctx context.Context, jobID string, now time.Time) (governance.LibraryJob, bool, error)
|
||||
ResetQueueJobForResume(ctx context.Context, queueJobID string, now time.Time) error
|
||||
}
|
||||
|
||||
func libraryStore(config Config) (libraryJobStore, bool) {
|
||||
|
||||
@@ -157,6 +157,26 @@ func (f *fakeLibraryStore) CancelLibraryJob(_ context.Context, _, _ string, _ ti
|
||||
return nil
|
||||
}
|
||||
|
||||
// 续作走原作业:只有 failed/cancelled 才能回到 pending。
|
||||
func (f *fakeLibraryStore) PrepareLibraryJobResume(_ context.Context, jobID string,
|
||||
_ time.Time) (governance.LibraryJob, bool, error) {
|
||||
for index, job := range f.jobs {
|
||||
if job.ID != jobID {
|
||||
continue
|
||||
}
|
||||
if job.State != governance.LibraryJobFailed && job.State != governance.LibraryJobCancelled {
|
||||
return job, false, nil
|
||||
}
|
||||
f.jobs[index].State = governance.LibraryJobPending
|
||||
return f.jobs[index], true, nil
|
||||
}
|
||||
return governance.LibraryJob{}, false, nil
|
||||
}
|
||||
|
||||
func (f *fakeLibraryStore) ResetQueueJobForResume(_ context.Context, _ string, _ time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestLibrarySyncGateRequiresEveryPieceOfEvidence(t *testing.T) {
|
||||
known := libraryKnownView{State: "known", Version: &libraryVersionView{ID: "v1"}}
|
||||
cases := []struct {
|
||||
|
||||
@@ -275,77 +275,58 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
// 幂等键按源作业固定:重复点击返回同一个续作作业,而不是每次新建。
|
||||
jobID := sourceID + "-resume"
|
||||
var kind governance.LibraryJobKind
|
||||
queueType := ""
|
||||
switch source.Kind {
|
||||
case governance.LibraryJobSync:
|
||||
kind, queueType = governance.LibraryJobSync, librarySyncJobType
|
||||
queueType = librarySyncJobType
|
||||
case governance.LibraryJobRestore:
|
||||
kind, queueType = governance.LibraryJobRestore, libraryRestoreJobType
|
||||
queueType = libraryRestoreJobType
|
||||
default:
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"error": "封存作业由封存入口重发,不走续作"})
|
||||
return
|
||||
}
|
||||
// F15:续作是**同一个意图的新执行代次**,不是新意图。沿用原作业的 operation,
|
||||
// 维护窗口、预备份、写集合的谱系都还挂在同一条意图上;新作业行只是这次执行的载体,
|
||||
// worker 起手会为它开一个更大的 epoch(旧代次从此不能写)。
|
||||
operationID := source.OperationID
|
||||
if operationID == "" {
|
||||
// 迁移前的老作业没有 operation:按原作业身份补一条(同键同摘要 ⇒ 幂等,
|
||||
// 不会因为点了两次续作而造出两条意图)。
|
||||
intent, intentErr := ensureLibraryIntent(r.Context(), store, source.Scope, kind,
|
||||
source.TargetSnapshotID, digest, identity.Username, "resume:"+sourceID, now)
|
||||
if intentErr != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register resume intent"})
|
||||
return
|
||||
}
|
||||
operationID = intent.Operation.ID
|
||||
}
|
||||
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
|
||||
ID: jobID, Scope: source.Scope, Kind: kind,
|
||||
IdempotencyKey: "library.job:" + operationID + ":resume",
|
||||
RequestDigest: digest, OperationID: operationID,
|
||||
TargetSnapshotID: source.TargetSnapshotID, Actor: identity.Username,
|
||||
}, governance.SyncStages(), now)
|
||||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"error": "该目标已有在飞作业"})
|
||||
return
|
||||
}
|
||||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"error": "同一续作请求参数不一致"})
|
||||
return
|
||||
}
|
||||
// R3:续作沿用**原业务作业**,只新增执行代次。
|
||||
//
|
||||
// 维护窗口 owner_job_id、freeze journal 文件名、预备份与写集合的身份全部按业务作业
|
||||
// id 落盘。另建 `-resume` 作业只能继承 operation,其它所有者仍指向旧作业:
|
||||
// AcquireInstanceMaintenance 会拒绝它、journal 与预备份也查不到——那样"续作"只是
|
||||
// 一个漂亮的新行,接不上原窗口。原作业 + 新 attempt 才是批准协议的形状。
|
||||
prepared, resumed, err := store.PrepareLibraryJobResume(r.Context(), sourceID, now)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create resumed job"})
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot prepare resumed job"})
|
||||
return
|
||||
}
|
||||
if reused {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "resumed_from": sourceID, "reused": true})
|
||||
if !resumed {
|
||||
// 状态在检查之后被别的执行者推进(已接管/已完成):不许再塞一次。
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_state_changed",
|
||||
"error": "作业已不在可续作状态(可能已被接管或完成)"})
|
||||
return
|
||||
}
|
||||
// 写入已完成、只差登记版本的情况:续作只做幂等收尾,不重写生产。
|
||||
body := map[string]any{"library_job_id": job.ID, "scope": source.Scope,
|
||||
body := map[string]any{"library_job_id": prepared.ID, "scope": source.Scope,
|
||||
"target_snapshot_id": source.TargetSnapshotID}
|
||||
if source.Stage == "finalize" && source.SnapshotID != "" {
|
||||
body["finalize_only"] = true
|
||||
}
|
||||
payload, _ := json.Marshal(body)
|
||||
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||||
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload, AvailableAt: now,
|
||||
MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + job.ID,
|
||||
}, now)
|
||||
if err != nil || queueJob == nil {
|
||||
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败", now)
|
||||
queueID := governance.QueueJobIDFor(prepared.ID)
|
||||
if _, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||||
ID: queueID, Type: queueType, Payload: payload, AvailableAt: now,
|
||||
MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + prepared.ID,
|
||||
}, now); err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue resumed job"})
|
||||
return
|
||||
}
|
||||
// 附不上队列号就等于作业没人跑:必须显式失败并取消,不能静默返回 202。
|
||||
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
|
||||
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列归属写入失败", now)
|
||||
// 队列行可能还是上一次的终态:重置为 pending。仍有效的租约 ⇒ 有执行在跑,让路。
|
||||
if resetErr := store.ResetQueueJobForResume(r.Context(), queueID, now); resetErr != nil {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_busy",
|
||||
"error": "该作业已有执行在跑(租约仍有效),等它结束再续作", "detail": errString(resetErr)})
|
||||
return
|
||||
}
|
||||
if attachErr := store.AttachLibraryQueueJob(r.Context(), prepared.ID, queueID, now); attachErr != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach resumed job"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "resumed_from": sourceID, "reused": false})
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": prepared, "resumed_from": sourceID,
|
||||
"resumed": true, "next_attempt_epoch": prepared.AttemptEpoch + 1})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user