diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b1c685..9af263f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,12 @@ 后续恢复请求全部被 `409 library_restore_active` 挡住。 现在「结束上一代 attempt + 业务作业置 pending + 队列行 upsert(含租约检查)」在同一个事务里完成: 入队失败整笔回滚、作业状态不变;租约仍有效时返回 `library_resume_busy` 而不是半态。 -- 续作改为拒绝 `stage=rollback` 的作业(`409 library_resume_requires_recover`):这类作业已经回滚并 - 保留维护窗口,正确出口是维护恢复(`/library/maintenance/recover`),它会绑定原 `window_job_id` - 并使用原 before-image;继续续作等于丢开前镜像重来。 +- 续作改为拒绝 `stage=rollback` 的作业(`409 library_resume_requires_recover`):这类作业已经回滚, + 正确出口是维护恢复(`/library/maintenance/recover`),它绑定原 `window_job_id` 并使用原 before-image; + 继续续作等于丢开前镜像重来。拒绝文案按维护记录如实给出窗口是否仍被持有,不一律宣称"仍持有"。 +- 阶段/状态/执行权与队列载荷全部在**事务内锁住的那一行**上校验并重建(`SELECT … FOR UPDATE`): + 调用方手里的旧快照不再参与决策,避免用过期 stage 复活已变化的作业;并发重复续作只有一个成功, + 另一个得到 `library_resume_busy` 或 `library_resume_state_changed`,不会重复投递。 ## 开发快照(发布页维护文案与版本记录页加载失败) diff --git a/internal/httpapi/library_flow_test.go b/internal/httpapi/library_flow_test.go index d1fd197..8face85 100644 --- a/internal/httpapi/library_flow_test.go +++ b/internal/httpapi/library_flow_test.go @@ -33,6 +33,10 @@ type fakeLibraryStore struct { // operations/lastOperation 记录"意图身份":重放与换键的差别在这里表达(F01)。 operations map[string]governance.LibraryOperation lastOperation governance.LibraryOperation + // 续作:resumeOK/resumeErr 驱动存储层结果,resumeCalls 记录每次调用(断言不重复投递)。 + resumeOK bool + resumeErr error + resumeCalls []string } func (f *fakeLibraryStore) CreateLibraryJob(_ context.Context, job governance.LibraryJob, _ []string, _ time.Time) (governance.LibraryJob, bool, error) { @@ -59,6 +63,11 @@ func (f *fakeLibraryStore) EnsureLibraryOperation(_ context.Context, op governan } func (f *fakeLibraryStore) LibraryJob(_ context.Context, id string) (governance.LibraryJob, error) { + for _, job := range f.jobs { + if job.ID == id { + return job, nil + } + } return governance.LibraryJob{ID: id}, nil } @@ -178,6 +187,23 @@ func (f *fakeLibraryStore) ResetQueueJobForResume(_ context.Context, _ string, _ return nil } +// ResumeLibraryJob 模拟存储层事务语义:结果由 resumeResult 驱动,调用次数可断言"重复请求不重复投递"。 +func (f *fakeLibraryStore) ResumeLibraryJob(_ context.Context, jobID, queueType string, _ time.Time) (governance.LibraryJob, bool, error) { + f.resumeCalls = append(f.resumeCalls, jobID+"/"+queueType) + if f.resumeErr != nil { + return governance.LibraryJob{}, false, f.resumeErr + } + if !f.resumeOK { + return governance.LibraryJob{}, false, nil + } + for _, job := range f.jobs { + if job.ID == jobID { + return job, true, nil + } + } + return governance.LibraryJob{}, false, nil +} + func TestLibrarySyncGateRequiresEveryPieceOfEvidence(t *testing.T) { known := libraryKnownView{State: "known", Version: &libraryVersionView{ID: "v1"}} cases := []struct { diff --git a/internal/httpapi/library_restore.go b/internal/httpapi/library_restore.go index 9ccf76c..a30e822 100644 --- a/internal/httpapi/library_restore.go +++ b/internal/httpapi/library_restore.go @@ -271,41 +271,31 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) { // id 落盘。另建 `-resume` 作业只能继承 operation,其它所有者仍指向旧作业: // AcquireInstanceMaintenance 会拒绝它、journal 与预备份也查不到——那样"续作"只是 // 一个漂亮的新行,接不上原窗口。原作业 + 新 attempt 才是批准协议的形状。 - // 已回滚并持有维护窗口的作业不能续作:继续跑等于丢开原 before-image 重来一遍, - // 正确出口是维护恢复(/library/maintenance/recover),它绑定原 window_job_id。 - if source.Stage == "rollback" { - writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_requires_recover", - "error": "作业已在回滚阶段失败并保留维护窗口,请走维护恢复而不是续作"}) - return - } - // 写入已完成、只差登记版本的情况:续作只做幂等收尾,不重写生产。 - body := map[string]any{"library_job_id": sourceID, "scope": source.Scope, - "target_snapshot_id": source.TargetSnapshotID} - if source.Stage == "finalize" && source.SnapshotID != "" { - body["finalize_only"] = true - } - payload, _ := json.Marshal(body) - queueID := governance.QueueJobIDFor(sourceID) - // 状态变更与队列 upsert 必须在同一事务里:否则入队失败会把作业留在 pending 却没有队列行。 + // 阶段/状态/执行权与载荷全部在存储层的事务里校验并构造:handler 手里的 source 只用于 + // 授权与闸口判断,不参与续作决策,避免用过期快照复活一个已经变化的作业。 resumer, ok := config.Store.(interface { - ResumeLibraryJob(ctx context.Context, jobID, queueID, queueType string, payload []byte, now time.Time) (governance.LibraryJob, bool, error) + ResumeLibraryJob(ctx context.Context, jobID, queueType string, now time.Time) (governance.LibraryJob, bool, error) }) if !ok { writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不支持续作"}) return } - prepared, resumed, err := resumer.ResumeLibraryJob(r.Context(), sourceID, queueID, queueType, payload, now) - if errors.Is(err, postgres.ErrLibraryResumeBusy) { + prepared, resumed, err := resumer.ResumeLibraryJob(r.Context(), sourceID, queueType, now) + switch { + case errors.Is(err, postgres.ErrLibraryResumeBusy): writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_busy", "error": "该作业已有执行在跑(租约仍有效),等它结束再续作"}) return - } - if err != nil { + case errors.Is(err, postgres.ErrLibraryResumeRequiresRecover): + // 窗口是否仍被持有由维护记录决定:已释放的回滚作业说"窗口已释放",不能一律宣称仍持有。 + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_requires_recover", + "error": "作业停在回滚阶段,请走维护恢复而不是续作", + "maintenance_held": maintenanceHeldFor(r.Context(), config, source)}) + return + case err != nil: writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot resume job"}) return - } - if !resumed { - // 状态在检查之后被别的执行者推进(已接管/已完成):不许再塞一次。 + case !resumed: writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_state_changed", "error": "作业已不在可续作状态(可能已被接管或完成)"}) return @@ -313,3 +303,15 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) { writeJSON(w, http.StatusAccepted, map[string]any{"job": prepared, "resumed_from": sourceID, "resumed": true, "next_attempt_epoch": prepared.AttemptEpoch + 1}) } + +// maintenanceHeldFor 只回答"这个作业现在是否仍持有维护窗口",用于给回滚作业的拒绝文案定性。 +func maintenanceHeldFor(ctx context.Context, config Config, job governance.LibraryJob) bool { + reader, ok := config.Store.(interface { + LibraryMaintenanceOf(ctx context.Context, scope string) (*postgres.LibraryMaintenance, error) + }) + if !ok { + return false + } + m, err := reader.LibraryMaintenanceOf(ctx, job.Scope) + return err == nil && m != nil && m.OwnerJobID == job.ID && m.ReleasedAt == nil +} diff --git a/internal/httpapi/library_resume_r26_test.go b/internal/httpapi/library_resume_r26_test.go new file mode 100644 index 0000000..b570d50 --- /dev/null +++ b/internal/httpapi/library_resume_r26_test.go @@ -0,0 +1,118 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/platform/postgres" +) + +func resumeRequest(id string) *http.Request { + request := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/"+id+"/resume", nil) + request.SetPathValue("id", id) + return request.WithContext(context.WithValue(request.Context(), sessionKey{}, + sessionIdentity{Username: "admin", Role: roleAdmin})) +} + +func resumableStore(job governance.LibraryJob) *fakeLibraryStore { + now := time.Now().UTC() + digest, _ := librarySyncParamsDigest(job.Scope, job.TargetSnapshotID) + if job.Kind == governance.LibraryJobRestore { + digest, _ = libraryRestoreParamsDigest(job.Scope, job.TargetSnapshotID) + } + store := &fakeLibraryStore{ + jobs: []governance.LibraryJob{job}, + grant: &governance.LibraryReleaseAuthorization{ID: "a", Scope: job.Scope, + SnapshotID: job.TargetSnapshotID, ParamsDigest: digest, ExpiresAt: now.Add(time.Hour), CreatedAt: now}, + } + store.resumeOK = true + return store +} + +func resumeConfig(store *fakeLibraryStore) Config { + return Config{Store: store, ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: true} +} + +// 合法续作:202,并且只投递一次。 +func TestR26ResumeAcceptedDeliversOnce(t *testing.T) { + job := governance.LibraryJob{ID: "job-ok", Scope: "internal", Kind: governance.LibraryJobSync, + State: governance.LibraryJobFailed, Stage: "resources", TargetSnapshotID: "snap-1"} + store := resumableStore(job) + recorder := httptest.NewRecorder() + libraryJobResume(recorder, resumeRequest("job-ok"), resumeConfig(store)) + if recorder.Code != http.StatusAccepted { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if len(store.resumeCalls) != 1 || store.resumeCalls[0] != "job-ok/library.sync" { + t.Fatalf("resume calls=%v", store.resumeCalls) + } +} + +// 回滚阶段的作业:409,且文案不能一律宣称"仍持有维护窗口"(窗口可能已释放)。 +func TestR26ResumeRollbackStageRequiresRecover(t *testing.T) { + job := governance.LibraryJob{ID: "job-rb", Scope: "internal", Kind: governance.LibraryJobRestore, + State: governance.LibraryJobFailed, Stage: "rollback", TargetSnapshotID: "snap-1"} + store := resumableStore(job) + store.resumeErr = postgres.ErrLibraryResumeRequiresRecover + recorder := httptest.NewRecorder() + libraryJobResume(recorder, resumeRequest("job-rb"), resumeConfig(store)) + if recorder.Code != http.StatusConflict || !strings.Contains(recorder.Body.String(), "library_resume_requires_recover") { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if strings.Contains(recorder.Body.String(), "保留维护窗口") { + t.Fatalf("must not always claim the window is held: %s", recorder.Body.String()) + } +} + +// 租约仍有效:409 busy,不是 500。 +func TestR26ResumeBusyIsConflict(t *testing.T) { + job := governance.LibraryJob{ID: "job-busy", Scope: "internal", Kind: governance.LibraryJobSync, + State: governance.LibraryJobFailed, Stage: "resources", TargetSnapshotID: "snap-1"} + store := resumableStore(job) + store.resumeErr = postgres.ErrLibraryResumeBusy + recorder := httptest.NewRecorder() + libraryJobResume(recorder, resumeRequest("job-busy"), resumeConfig(store)) + if recorder.Code != http.StatusConflict || !strings.Contains(recorder.Body.String(), "library_resume_busy") { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } +} + +// 入队失败(存储层事务回滚):500,且不得把作业当成已续作。 +func TestR26ResumeEnqueueFailureIsServerError(t *testing.T) { + job := governance.LibraryJob{ID: "job-fail", Scope: "internal", Kind: governance.LibraryJobRestore, + State: governance.LibraryJobFailed, Stage: "resources", TargetSnapshotID: "snap-1"} + store := resumableStore(job) + store.resumeErr = errors.New("enqueue resumed job: injected") + recorder := httptest.NewRecorder() + libraryJobResume(recorder, resumeRequest("job-fail"), resumeConfig(store)) + if recorder.Code != http.StatusInternalServerError { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if strings.Contains(recorder.Body.String(), "resumed") { + t.Fatalf("failed enqueue must not claim success: %s", recorder.Body.String()) + } +} + +// 重复请求:第二次被存储层拒绝(状态已推进),不会重复投递。 +func TestR26ResumeDuplicateRequestDoesNotRedeliver(t *testing.T) { + job := governance.LibraryJob{ID: "job-dup", Scope: "internal", Kind: governance.LibraryJobSync, + State: governance.LibraryJobFailed, Stage: "resources", TargetSnapshotID: "snap-1"} + store := resumableStore(job) + first := httptest.NewRecorder() + libraryJobResume(first, resumeRequest("job-dup"), resumeConfig(store)) + store.resumeOK = false // 第二次:业务作业已不是 failed + second := httptest.NewRecorder() + libraryJobResume(second, resumeRequest("job-dup"), resumeConfig(store)) + if first.Code != http.StatusAccepted || second.Code != http.StatusConflict { + t.Fatalf("first=%d second=%d body=%s", first.Code, second.Code, second.Body.String()) + } + if len(store.resumeCalls) != 2 { + t.Fatalf("expected 2 store calls, got %v", store.resumeCalls) + } +} diff --git a/internal/platform/postgres/library_jobs.go b/internal/platform/postgres/library_jobs.go index 593179a..7a4c3fa 100644 --- a/internal/platform/postgres/library_jobs.go +++ b/internal/platform/postgres/library_jobs.go @@ -505,25 +505,47 @@ WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC()) // ErrLibraryResumeBusy 表示队列里那一代仍持有有效租约,续作必须让路。 var ErrLibraryResumeBusy = errors.New("library resume busy") -// ResumeLibraryJob 在同一事务里完成续作所需的全部持久变更: -// 结束上一代 attempt、把业务作业置回 pending、把队列行 upsert 成 pending。 +// ErrLibraryResumeRequiresRecover 表示作业停在回滚阶段:它已经回退并(可能)保留维护窗口, +// 正确出口是维护恢复,而不是按原参数再来一遍。 +var ErrLibraryResumeRequiresRecover = errors.New("library resume requires recover") + +// ResumeLibraryJob 在同一事务里完成续作所需的全部持久变更,并用**事务内锁住的那一行**构造队列载荷: +// 结束上一代 attempt → 校验状态/阶段/执行权 → 业务作业置 pending → 队列行 upsert(含租约检查)。 // -// 为什么必须放进一个事务:旧实现先 PrepareLibraryJobResume 提交 state='pending',再由 API 单独 -// Enqueue。队列行的主键由业务作业 id 派生(QueueJobIDFor),第二次续作时 INSERT 会撞主键—— -// 入队失败,但 state 已经变成 pending 且没有队列行,作业就永久卡在"看起来在跑"的假 active 上, -// 后续恢复全部被 409 挡住。状态与队列必须同生共死。 -func (s *Store) ResumeLibraryJob(ctx context.Context, jobID, queueID, queueType string, payload []byte, now time.Time) (governance.LibraryJob, bool, error) { +// 为什么必须一个事务 + 行锁:旧实现先把 state 提交成 pending,再由 API 单独 Enqueue;队列行主键由 +// 业务作业 id 派生,第二次续作时 INSERT 撞主键——入队失败但状态已改,作业永久卡在"假 active"。 +// 同时,阶段与载荷从锁内那一行读,调用方手里的旧快照不会被用来复活一个已经变化的作业。 +func (s *Store) ResumeLibraryJob(ctx context.Context, jobID, queueType string, now time.Time) (governance.LibraryJob, bool, error) { tx, err := s.db.Begin(ctx) if err != nil { return governance.LibraryJob{}, false, fmt.Errorf("resume library job: %w", err) } defer func() { _ = tx.Rollback(ctx) }() + var state, stage, kind, scope, targetSnapshot, snapshotID string + var attemptEpoch int + if err := tx.QueryRow(ctx, ` +SELECT state, stage, kind, scope, target_snapshot_id, snapshot_id, attempt_epoch +FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan( + &state, &stage, &kind, &scope, &targetSnapshot, &snapshotID, &attemptEpoch); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryJob{}, false, err + } + return governance.LibraryJob{}, false, fmt.Errorf("lock library job for resume: %w", err) + } + if state != string(governance.LibraryJobFailed) && state != string(governance.LibraryJobCancelled) { + // 状态已被别的执行者推进(接管/完成):不落任何变更。 + return governance.LibraryJob{}, false, nil + } + if stage == "rollback" { + return governance.LibraryJob{}, false, ErrLibraryResumeRequiresRecover + } if _, err := tx.Exec(ctx, ` UPDATE library_attempts SET state='abandoned', finished_at=$2, outcome='superseded by operator resume', lease_until=NULL WHERE job_id=$1 AND state IN ('claimed','running')`, jobID, now.UTC()); err != nil { return governance.LibraryJob{}, false, fmt.Errorf("retire attempts for resume: %w", err) } + queueID := governance.QueueJobIDFor(jobID) tag, err := tx.Exec(ctx, ` UPDATE library_jobs SET state='pending', error='', recovery='', finished_at=NULL, updated_at=$2, lease_owner='', lease_until=NULL, queue_job_id=$3 @@ -534,7 +556,7 @@ WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC(), queueID) if tag.RowsAffected() != 1 { return governance.LibraryJob{}, false, nil } - // 队列行可能已存在(上一次执行)。先锁住它判断租约:有活跃执行者就整笔回滚,绝不留半态。 + // 队列行可能已存在(上一次执行)。锁住它判断租约:有活跃执行者就整笔回滚。 var lease *time.Time row := tx.QueryRow(ctx, `SELECT lease_until FROM jobs WHERE id=$1 FOR UPDATE`, queueID) if err := row.Scan(&lease); err != nil && !errors.Is(err, pgx.ErrNoRows) { @@ -543,6 +565,15 @@ WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC(), queueID) if lease != nil && lease.After(now.UTC()) { return governance.LibraryJob{}, false, ErrLibraryResumeBusy } + // 载荷按锁内那一行重建,不用调用方可能过期的快照。 + body := map[string]any{"library_job_id": jobID, "scope": scope, "target_snapshot_id": targetSnapshot} + if stage == "finalize" && snapshotID != "" { + body["finalize_only"] = true + } + payload, err := json.Marshal(body) + if err != nil { + return governance.LibraryJob{}, false, fmt.Errorf("encode resumed payload: %w", err) + } if _, err := tx.Exec(ctx, ` INSERT INTO jobs (id, type, payload, state, priority, available_at, max_attempts, idempotency_key, created_at, updated_at) VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $5, $5) diff --git a/internal/platform/postgres/library_resume_r25_test.go b/internal/platform/postgres/library_resume_r25_test.go index 84365c5..ad611c3 100644 --- a/internal/platform/postgres/library_resume_r25_test.go +++ b/internal/platform/postgres/library_resume_r25_test.go @@ -2,6 +2,7 @@ package postgres import ( "context" + "encoding/json" "errors" "github.com/jackc/pgx/v5/pgxpool" @@ -41,9 +42,9 @@ func TestR25ResumeReusesExistingQueueRowAtomically(t *testing.T) { store, pool := newIdentityStore(t) jobID := "librestore-internal-r25-atomic" past := time.Now().UTC().Add(-time.Minute) - queueID := seedResumableJob(t, pool, jobID, "rollback", "failed", &past) + queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &past) - job, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, queueID, "library.restore", []byte(`{}`), time.Now().UTC()) + job, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC()) if err != nil { t.Fatalf("resume with existing queue row: %v", err) } @@ -70,9 +71,9 @@ func TestR25ResumeBusyLeaseDoesNotMutateState(t *testing.T) { store, pool := newIdentityStore(t) jobID := "librestore-internal-r25-busy" future := time.Now().UTC().Add(10 * time.Minute) - queueID := seedResumableJob(t, pool, jobID, "rollback", "failed", &future) + queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &future) - _, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, queueID, "library.restore", []byte(`{}`), time.Now().UTC()) + _, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC()) if !errors.Is(err, ErrLibraryResumeBusy) { t.Fatalf("want ErrLibraryResumeBusy, got %v (resumed=%v)", err, resumed) } @@ -81,7 +82,7 @@ func TestR25ResumeBusyLeaseDoesNotMutateState(t *testing.T) { `SELECT state, stage FROM library_jobs WHERE id=$1`, jobID).Scan(&state, &stage); err != nil { t.Fatalf("reload: %v", err) } - if state != "failed" || stage != "rollback" { + if state != "failed" || stage != "resources" { t.Fatalf("busy resume mutated job: state=%s stage=%s", state, stage) } var qState string @@ -97,10 +98,9 @@ func TestR25ResumeBusyLeaseDoesNotMutateState(t *testing.T) { func TestR25ResumeRejectsNonFailedStateWithoutMutation(t *testing.T) { store, pool := newIdentityStore(t) jobID := "librestore-internal-r25-succeeded" - past := time.Now().UTC().Add(-time.Minute) - queueID := seedResumableJob(t, pool, jobID, "finalize", "succeeded", &past) + seedResumableJob(t, pool, jobID, "finalize", "succeeded", ptrTime(time.Now().UTC().Add(-time.Minute))) - _, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, queueID, "library.restore", []byte(`{}`), time.Now().UTC()) + _, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC()) if err != nil { t.Fatalf("resume on succeeded job: %v", err) } @@ -115,3 +115,143 @@ func TestR25ResumeRejectsNonFailedStateWithoutMutation(t *testing.T) { t.Fatalf("state changed on rejected resume: %s", state) } } + +// 入队真的失败(数据库层拒绝写队列)时,attempt / job / queue 必须全部回到原值: +// 不能留下"state=pending 却没有队列行"的假 active。 +func TestR26ResumeEnqueueFailureRollsBackEveryField(t *testing.T) { + store, pool := newIdentityStore(t) + ctx := context.Background() + jobID := "librestore-internal-r26-rollback-all" + past := time.Now().UTC().Add(-time.Minute) + queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &past) + if _, err := pool.Exec(ctx, `UPDATE library_jobs SET error='boom', recovery='rolled back', + lease_owner='old', lease_until=$2, attempt_epoch=3 WHERE id=$1`, jobID, past.Add(30*time.Second)); err != nil { + t.Fatalf("seed fields: %v", err) + } + before := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID) + queueBefore := rowJSON(t, pool, `SELECT to_jsonb(j) FROM jobs j WHERE id=$1`, queueID) + if _, err := pool.Exec(ctx, `CREATE FUNCTION r26_reject_queue() RETURNS trigger AS $$ BEGIN + RAISE EXCEPTION 'r26 injected queue write failure'; END; $$ LANGUAGE plpgsql; + CREATE TRIGGER r26_reject BEFORE INSERT OR UPDATE ON jobs FOR EACH ROW EXECUTE FUNCTION r26_reject_queue();`); err != nil { + t.Fatalf("install trigger: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DROP TRIGGER IF EXISTS r26_reject ON jobs; + DROP FUNCTION IF EXISTS r26_reject_queue();`) + }) + if _, resumed, err := store.ResumeLibraryJob(ctx, jobID, "library.restore", time.Now().UTC()); err == nil || resumed { + t.Fatalf("resume must fail when queue write is rejected: resumed=%v err=%v", resumed, err) + } + if after := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID); after != before { + t.Fatalf("library_jobs changed after failed enqueue:\n before=%s\n after=%s", before, after) + } + if after := rowJSON(t, pool, `SELECT to_jsonb(j) FROM jobs j WHERE id=$1`, queueID); after != queueBefore { + t.Fatalf("queue row changed after failed enqueue:\n before=%s\n after=%s", queueBefore, after) + } +} + +// 回滚阶段的作业必须在事务内被拒绝(不是靠 handler 读到的旧快照)。 +func TestR26ResumeRejectsRollbackStageInsideTransaction(t *testing.T) { + store, pool := newIdentityStore(t) + jobID := "librestore-internal-r26-rollback-stage" + seedResumableJob(t, pool, jobID, "rollback", "failed", ptrTime(time.Now().UTC().Add(-time.Minute))) + before := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID) + _, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC()) + if !errors.Is(err, ErrLibraryResumeRequiresRecover) || resumed { + t.Fatalf("want ErrLibraryResumeRequiresRecover, got resumed=%v err=%v", resumed, err) + } + if after := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID); after != before { + t.Fatalf("rollback rejection mutated job:\n before=%s\n after=%s", before, after) + } +} + +// 并发重复续作只允许一个成功,队列只有一条投递(不能双投)。 +func TestR26ResumeConcurrentDuplicateDeliversOnce(t *testing.T) { + store, pool := newIdentityStore(t) + jobID := "librestore-internal-r26-concurrent" + past := time.Now().UTC().Add(-time.Minute) + queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &past) + + type res struct { + resumed bool + err error + } + ch := make(chan res, 2) + start := make(chan struct{}) + for i := 0; i < 2; i++ { + go func() { + <-start + _, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC()) + ch <- res{resumed, err} + }() + } + close(start) + ok, busy := 0, 0 + for i := 0; i < 2; i++ { + r := <-ch + switch { + case r.resumed && r.err == nil: + ok++ + case errors.Is(r.err, ErrLibraryResumeBusy) || (r.err == nil && !r.resumed): + busy++ + case r.err != nil: + t.Fatalf("unexpected error: %v", r.err) + } + } + if ok != 1 || busy != 1 { + t.Fatalf("want exactly one resume and one refusal, got ok=%d refused=%d", ok, busy) + } + var state string + var attempt int + if err := pool.QueryRow(context.Background(), `SELECT state, attempt FROM jobs WHERE id=$1`, queueID).Scan(&state, &attempt); err != nil { + t.Fatalf("queue: %v", err) + } + if state != "pending" || attempt != 0 { + t.Fatalf("queue not reset exactly once: state=%s attempt=%d", state, attempt) + } + var pending int + if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM library_jobs WHERE id=$1 AND state='pending'`, jobID).Scan(&pending); err != nil { + t.Fatalf("job: %v", err) + } + if pending != 1 { + t.Fatalf("job not pending: %d", pending) + } +} + +// 载荷必须来自事务内锁定行:finalize 阶段续作要带 finalize_only,而不是调用方旧快照。 +func TestR26ResumePayloadComesFromLockedRow(t *testing.T) { + store, pool := newIdentityStore(t) + ctx := context.Background() + jobID := "librestore-internal-r26-finalize" + past := time.Now().UTC().Add(-time.Minute) + queueID := seedResumableJob(t, pool, jobID, "finalize", "failed", &past) + if _, err := pool.Exec(ctx, `UPDATE library_jobs SET snapshot_id='snap-written' WHERE id=$1`, jobID); err != nil { + t.Fatalf("seed snapshot: %v", err) + } + if _, resumed, err := store.ResumeLibraryJob(ctx, jobID, "library.restore", time.Now().UTC()); err != nil || !resumed { + t.Fatalf("resume: resumed=%v err=%v", resumed, err) + } + var payload []byte + if err := pool.QueryRow(ctx, `SELECT payload FROM jobs WHERE id=$1`, queueID).Scan(&payload); err != nil { + t.Fatalf("payload: %v", err) + } + var decoded map[string]any + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("payload json: %v", err) + } + if decoded["finalize_only"] != true || decoded["target_snapshot_id"] != "snap-target" { + t.Fatalf("payload not built from locked row: %v", decoded) + } +} + +// rowJSON 取整行快照(列顺序无关),用于"逐字段前后完全一致"的断言。 +func rowJSON(t *testing.T, pool *pgxpool.Pool, query, id string) string { + t.Helper() + var s string + if err := pool.QueryRow(context.Background(), query, id).Scan(&s); err != nil { + t.Fatalf("row json: %v", err) + } + return s +} + +func ptrTime(t time.Time) *time.Time { return &t }