@@ -505,25 +505,47 @@ WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC())
|
||||
// ErrLibraryResumeBusy 表示队列里那一代仍持有有效租约,续作必须让路。
|
||||
var ErrLibraryResumeBusy = errors.New("library resume busy")
|
||||
|
||||
// ResumeLibraryJob 在同一事务里完成续作所需的全部持久变更:
|
||||
// 结束上一代 attempt、把业务作业置回 pending、把队列行 upsert 成 pending。
|
||||
// ErrLibraryResumeRequiresRecover 表示作业停在回滚阶段:它已经回退并(可能)保留维护窗口,
|
||||
// 正确出口是维护恢复,而不是按原参数再来一遍。
|
||||
var ErrLibraryResumeRequiresRecover = errors.New("library resume requires recover")
|
||||
|
||||
// ResumeLibraryJob 在同一事务里完成续作所需的全部持久变更,并用**事务内锁住的那一行**构造队列载荷:
|
||||
// 结束上一代 attempt → 校验状态/阶段/执行权 → 业务作业置 pending → 队列行 upsert(含租约检查)。
|
||||
//
|
||||
// 为什么必须放进一个事务:旧实现先 PrepareLibraryJobResume 提交 state='pending',再由 API 单独
|
||||
// Enqueue。队列行的主键由业务作业 id 派生(QueueJobIDFor),第二次续作时 INSERT 会撞主键——
|
||||
// 入队失败,但 state 已经变成 pending 且没有队列行,作业就永久卡在"看起来在跑"的假 active 上,
|
||||
// 后续恢复全部被 409 挡住。状态与队列必须同生共死。
|
||||
func (s *Store) ResumeLibraryJob(ctx context.Context, jobID, queueID, queueType string, payload []byte, now time.Time) (governance.LibraryJob, bool, error) {
|
||||
// 为什么必须一个事务 + 行锁:旧实现先把 state 提交成 pending,再由 API 单独 Enqueue;队列行主键由
|
||||
// 业务作业 id 派生,第二次续作时 INSERT 撞主键——入队失败但状态已改,作业永久卡在"假 active"。
|
||||
// 同时,阶段与载荷从锁内那一行读,调用方手里的旧快照不会被用来复活一个已经变化的作业。
|
||||
func (s *Store) ResumeLibraryJob(ctx context.Context, jobID, queueType string, now time.Time) (governance.LibraryJob, bool, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("resume library job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var state, stage, kind, scope, targetSnapshot, snapshotID string
|
||||
var attemptEpoch int
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT state, stage, kind, scope, target_snapshot_id, snapshot_id, attempt_epoch
|
||||
FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan(
|
||||
&state, &stage, &kind, &scope, &targetSnapshot, &snapshotID, &attemptEpoch); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.LibraryJob{}, false, err
|
||||
}
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("lock library job for resume: %w", err)
|
||||
}
|
||||
if state != string(governance.LibraryJobFailed) && state != string(governance.LibraryJobCancelled) {
|
||||
// 状态已被别的执行者推进(接管/完成):不落任何变更。
|
||||
return governance.LibraryJob{}, false, nil
|
||||
}
|
||||
if stage == "rollback" {
|
||||
return governance.LibraryJob{}, false, ErrLibraryResumeRequiresRecover
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_attempts SET state='abandoned', finished_at=$2, outcome='superseded by operator resume',
|
||||
lease_until=NULL
|
||||
WHERE job_id=$1 AND state IN ('claimed','running')`, jobID, now.UTC()); err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("retire attempts for resume: %w", err)
|
||||
}
|
||||
queueID := governance.QueueJobIDFor(jobID)
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE library_jobs SET state='pending', error='', recovery='', finished_at=NULL, updated_at=$2,
|
||||
lease_owner='', lease_until=NULL, queue_job_id=$3
|
||||
@@ -534,7 +556,7 @@ WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC(), queueID)
|
||||
if tag.RowsAffected() != 1 {
|
||||
return governance.LibraryJob{}, false, nil
|
||||
}
|
||||
// 队列行可能已存在(上一次执行)。先锁住它判断租约:有活跃执行者就整笔回滚,绝不留半态。
|
||||
// 队列行可能已存在(上一次执行)。锁住它判断租约:有活跃执行者就整笔回滚。
|
||||
var lease *time.Time
|
||||
row := tx.QueryRow(ctx, `SELECT lease_until FROM jobs WHERE id=$1 FOR UPDATE`, queueID)
|
||||
if err := row.Scan(&lease); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
@@ -543,6 +565,15 @@ WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC(), queueID)
|
||||
if lease != nil && lease.After(now.UTC()) {
|
||||
return governance.LibraryJob{}, false, ErrLibraryResumeBusy
|
||||
}
|
||||
// 载荷按锁内那一行重建,不用调用方可能过期的快照。
|
||||
body := map[string]any{"library_job_id": jobID, "scope": scope, "target_snapshot_id": targetSnapshot}
|
||||
if stage == "finalize" && snapshotID != "" {
|
||||
body["finalize_only"] = true
|
||||
}
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("encode resumed payload: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO jobs (id, type, payload, state, priority, available_at, max_attempts, idempotency_key, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $5, $5)
|
||||
|
||||
@@ -2,6 +2,7 @@ package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -41,9 +42,9 @@ func TestR25ResumeReusesExistingQueueRowAtomically(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
jobID := "librestore-internal-r25-atomic"
|
||||
past := time.Now().UTC().Add(-time.Minute)
|
||||
queueID := seedResumableJob(t, pool, jobID, "rollback", "failed", &past)
|
||||
queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &past)
|
||||
|
||||
job, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, queueID, "library.restore", []byte(`{}`), time.Now().UTC())
|
||||
job, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
||||
if err != nil {
|
||||
t.Fatalf("resume with existing queue row: %v", err)
|
||||
}
|
||||
@@ -70,9 +71,9 @@ func TestR25ResumeBusyLeaseDoesNotMutateState(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
jobID := "librestore-internal-r25-busy"
|
||||
future := time.Now().UTC().Add(10 * time.Minute)
|
||||
queueID := seedResumableJob(t, pool, jobID, "rollback", "failed", &future)
|
||||
queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &future)
|
||||
|
||||
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, queueID, "library.restore", []byte(`{}`), time.Now().UTC())
|
||||
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
||||
if !errors.Is(err, ErrLibraryResumeBusy) {
|
||||
t.Fatalf("want ErrLibraryResumeBusy, got %v (resumed=%v)", err, resumed)
|
||||
}
|
||||
@@ -81,7 +82,7 @@ func TestR25ResumeBusyLeaseDoesNotMutateState(t *testing.T) {
|
||||
`SELECT state, stage FROM library_jobs WHERE id=$1`, jobID).Scan(&state, &stage); err != nil {
|
||||
t.Fatalf("reload: %v", err)
|
||||
}
|
||||
if state != "failed" || stage != "rollback" {
|
||||
if state != "failed" || stage != "resources" {
|
||||
t.Fatalf("busy resume mutated job: state=%s stage=%s", state, stage)
|
||||
}
|
||||
var qState string
|
||||
@@ -97,10 +98,9 @@ func TestR25ResumeBusyLeaseDoesNotMutateState(t *testing.T) {
|
||||
func TestR25ResumeRejectsNonFailedStateWithoutMutation(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
jobID := "librestore-internal-r25-succeeded"
|
||||
past := time.Now().UTC().Add(-time.Minute)
|
||||
queueID := seedResumableJob(t, pool, jobID, "finalize", "succeeded", &past)
|
||||
seedResumableJob(t, pool, jobID, "finalize", "succeeded", ptrTime(time.Now().UTC().Add(-time.Minute)))
|
||||
|
||||
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, queueID, "library.restore", []byte(`{}`), time.Now().UTC())
|
||||
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
||||
if err != nil {
|
||||
t.Fatalf("resume on succeeded job: %v", err)
|
||||
}
|
||||
@@ -115,3 +115,143 @@ func TestR25ResumeRejectsNonFailedStateWithoutMutation(t *testing.T) {
|
||||
t.Fatalf("state changed on rejected resume: %s", state)
|
||||
}
|
||||
}
|
||||
|
||||
// 入队真的失败(数据库层拒绝写队列)时,attempt / job / queue 必须全部回到原值:
|
||||
// 不能留下"state=pending 却没有队列行"的假 active。
|
||||
func TestR26ResumeEnqueueFailureRollsBackEveryField(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
jobID := "librestore-internal-r26-rollback-all"
|
||||
past := time.Now().UTC().Add(-time.Minute)
|
||||
queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &past)
|
||||
if _, err := pool.Exec(ctx, `UPDATE library_jobs SET error='boom', recovery='rolled back',
|
||||
lease_owner='old', lease_until=$2, attempt_epoch=3 WHERE id=$1`, jobID, past.Add(30*time.Second)); err != nil {
|
||||
t.Fatalf("seed fields: %v", err)
|
||||
}
|
||||
before := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID)
|
||||
queueBefore := rowJSON(t, pool, `SELECT to_jsonb(j) FROM jobs j WHERE id=$1`, queueID)
|
||||
if _, err := pool.Exec(ctx, `CREATE FUNCTION r26_reject_queue() RETURNS trigger AS $$ BEGIN
|
||||
RAISE EXCEPTION 'r26 injected queue write failure'; END; $$ LANGUAGE plpgsql;
|
||||
CREATE TRIGGER r26_reject BEFORE INSERT OR UPDATE ON jobs FOR EACH ROW EXECUTE FUNCTION r26_reject_queue();`); err != nil {
|
||||
t.Fatalf("install trigger: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), `DROP TRIGGER IF EXISTS r26_reject ON jobs;
|
||||
DROP FUNCTION IF EXISTS r26_reject_queue();`)
|
||||
})
|
||||
if _, resumed, err := store.ResumeLibraryJob(ctx, jobID, "library.restore", time.Now().UTC()); err == nil || resumed {
|
||||
t.Fatalf("resume must fail when queue write is rejected: resumed=%v err=%v", resumed, err)
|
||||
}
|
||||
if after := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID); after != before {
|
||||
t.Fatalf("library_jobs changed after failed enqueue:\n before=%s\n after=%s", before, after)
|
||||
}
|
||||
if after := rowJSON(t, pool, `SELECT to_jsonb(j) FROM jobs j WHERE id=$1`, queueID); after != queueBefore {
|
||||
t.Fatalf("queue row changed after failed enqueue:\n before=%s\n after=%s", queueBefore, after)
|
||||
}
|
||||
}
|
||||
|
||||
// 回滚阶段的作业必须在事务内被拒绝(不是靠 handler 读到的旧快照)。
|
||||
func TestR26ResumeRejectsRollbackStageInsideTransaction(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
jobID := "librestore-internal-r26-rollback-stage"
|
||||
seedResumableJob(t, pool, jobID, "rollback", "failed", ptrTime(time.Now().UTC().Add(-time.Minute)))
|
||||
before := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID)
|
||||
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
||||
if !errors.Is(err, ErrLibraryResumeRequiresRecover) || resumed {
|
||||
t.Fatalf("want ErrLibraryResumeRequiresRecover, got resumed=%v err=%v", resumed, err)
|
||||
}
|
||||
if after := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID); after != before {
|
||||
t.Fatalf("rollback rejection mutated job:\n before=%s\n after=%s", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// 并发重复续作只允许一个成功,队列只有一条投递(不能双投)。
|
||||
func TestR26ResumeConcurrentDuplicateDeliversOnce(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
jobID := "librestore-internal-r26-concurrent"
|
||||
past := time.Now().UTC().Add(-time.Minute)
|
||||
queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &past)
|
||||
|
||||
type res struct {
|
||||
resumed bool
|
||||
err error
|
||||
}
|
||||
ch := make(chan res, 2)
|
||||
start := make(chan struct{})
|
||||
for i := 0; i < 2; i++ {
|
||||
go func() {
|
||||
<-start
|
||||
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
||||
ch <- res{resumed, err}
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
ok, busy := 0, 0
|
||||
for i := 0; i < 2; i++ {
|
||||
r := <-ch
|
||||
switch {
|
||||
case r.resumed && r.err == nil:
|
||||
ok++
|
||||
case errors.Is(r.err, ErrLibraryResumeBusy) || (r.err == nil && !r.resumed):
|
||||
busy++
|
||||
case r.err != nil:
|
||||
t.Fatalf("unexpected error: %v", r.err)
|
||||
}
|
||||
}
|
||||
if ok != 1 || busy != 1 {
|
||||
t.Fatalf("want exactly one resume and one refusal, got ok=%d refused=%d", ok, busy)
|
||||
}
|
||||
var state string
|
||||
var attempt int
|
||||
if err := pool.QueryRow(context.Background(), `SELECT state, attempt FROM jobs WHERE id=$1`, queueID).Scan(&state, &attempt); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
if state != "pending" || attempt != 0 {
|
||||
t.Fatalf("queue not reset exactly once: state=%s attempt=%d", state, attempt)
|
||||
}
|
||||
var pending int
|
||||
if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM library_jobs WHERE id=$1 AND state='pending'`, jobID).Scan(&pending); err != nil {
|
||||
t.Fatalf("job: %v", err)
|
||||
}
|
||||
if pending != 1 {
|
||||
t.Fatalf("job not pending: %d", pending)
|
||||
}
|
||||
}
|
||||
|
||||
// 载荷必须来自事务内锁定行:finalize 阶段续作要带 finalize_only,而不是调用方旧快照。
|
||||
func TestR26ResumePayloadComesFromLockedRow(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
ctx := context.Background()
|
||||
jobID := "librestore-internal-r26-finalize"
|
||||
past := time.Now().UTC().Add(-time.Minute)
|
||||
queueID := seedResumableJob(t, pool, jobID, "finalize", "failed", &past)
|
||||
if _, err := pool.Exec(ctx, `UPDATE library_jobs SET snapshot_id='snap-written' WHERE id=$1`, jobID); err != nil {
|
||||
t.Fatalf("seed snapshot: %v", err)
|
||||
}
|
||||
if _, resumed, err := store.ResumeLibraryJob(ctx, jobID, "library.restore", time.Now().UTC()); err != nil || !resumed {
|
||||
t.Fatalf("resume: resumed=%v err=%v", resumed, err)
|
||||
}
|
||||
var payload []byte
|
||||
if err := pool.QueryRow(ctx, `SELECT payload FROM jobs WHERE id=$1`, queueID).Scan(&payload); err != nil {
|
||||
t.Fatalf("payload: %v", err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("payload json: %v", err)
|
||||
}
|
||||
if decoded["finalize_only"] != true || decoded["target_snapshot_id"] != "snap-target" {
|
||||
t.Fatalf("payload not built from locked row: %v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
// rowJSON 取整行快照(列顺序无关),用于"逐字段前后完全一致"的断言。
|
||||
func rowJSON(t *testing.T, pool *pgxpool.Pool, query, id string) string {
|
||||
t.Helper()
|
||||
var s string
|
||||
if err := pool.QueryRow(context.Background(), query, id).Scan(&s); err != nil {
|
||||
t.Fatalf("row json: %v", err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func ptrTime(t time.Time) *time.Time { return &t }
|
||||
|
||||
Reference in New Issue
Block a user