258 lines
11 KiB
Go
258 lines
11 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"testing"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
)
|
|
|
|
// seedResumableJob 建一条 failed/rollback 的库作业,并按 QueueJobIDFor 预置"上一次执行"留下的队列行,
|
|
// 复现线上真实形态:队列行主键已存在、idempotency_key 与续作不同(因此旧实现 INSERT 撞主键)。
|
|
func seedResumableJob(t *testing.T, pool *pgxpool.Pool, jobID, stage, state string, queueLease *time.Time) string {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
now := time.Now().UTC()
|
|
if _, err := pool.Exec(ctx, `
|
|
INSERT INTO library_jobs (id, scope, kind, state, stage, idempotency_key, request_digest, request,
|
|
queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, created_at, updated_at, operation_id)
|
|
VALUES ($1::text,'internal','restore',$2::text,$3::text,'idem-'||$1::text,'digest-'||$1::text,'{}'::jsonb,
|
|
$4::text,'snap-before','snap-target','tester','boom','', $5::timestamptz, $5::timestamptz, 'op-'||$1::text)`,
|
|
jobID, state, stage, governance.QueueJobIDFor(jobID), now); err != nil {
|
|
t.Fatalf("seed library job: %v", err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `
|
|
INSERT INTO jobs (id, type, payload, state, priority, available_at, max_attempts, idempotency_key,
|
|
created_at, updated_at, lease_owner, lease_until, attempt)
|
|
VALUES ($1::text,'library.restore','{}'::jsonb,'failed',0,$2::timestamptz,1,'library.restore.queue:'||$3::text,$2::timestamptz,$2::timestamptz,'old-worker',$4::timestamptz,1)`,
|
|
governance.QueueJobIDFor(jobID), now, jobID, queueLease); err != nil {
|
|
t.Fatalf("seed queue job: %v", err)
|
|
}
|
|
return governance.QueueJobIDFor(jobID)
|
|
}
|
|
|
|
// RED 的真实形态:队列行已存在时,续作必须成功复用该行;旧实现会在 Enqueue 撞主键后
|
|
// 留下 state=pending 且没有可用队列行的假 active。
|
|
func TestR25ResumeReusesExistingQueueRowAtomically(t *testing.T) {
|
|
store, pool := newIdentityStore(t)
|
|
jobID := "librestore-internal-r25-atomic"
|
|
past := time.Now().UTC().Add(-time.Minute)
|
|
queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &past)
|
|
|
|
job, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
|
if err != nil {
|
|
t.Fatalf("resume with existing queue row: %v", err)
|
|
}
|
|
if !resumed || job.State != governance.LibraryJobPending {
|
|
t.Fatalf("resume did not take effect: resumed=%v state=%s", resumed, job.State)
|
|
}
|
|
row := pool.QueryRow(context.Background(),
|
|
`SELECT state, attempt, coalesce(lease_owner,'') FROM jobs WHERE id=$1`, queueID)
|
|
var state, owner string
|
|
var attempt int
|
|
if err := row.Scan(&state, &attempt, &owner); err != nil {
|
|
t.Fatalf("queue row missing after resume: %v", err)
|
|
}
|
|
if state != "pending" || attempt != 0 || owner != "" {
|
|
t.Fatalf("queue row not reset for resume: state=%s attempt=%d owner=%q", state, attempt, owner)
|
|
}
|
|
if job.QueueJobID != queueID {
|
|
t.Fatalf("library job not attached to queue row: %q", job.QueueJobID)
|
|
}
|
|
}
|
|
|
|
// 有有效租约时必须整笔回滚:不能出现"业务作业 pending、队列仍被别人持有"的半态。
|
|
func TestR25ResumeBusyLeaseDoesNotMutateState(t *testing.T) {
|
|
store, pool := newIdentityStore(t)
|
|
jobID := "librestore-internal-r25-busy"
|
|
future := time.Now().UTC().Add(10 * time.Minute)
|
|
queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &future)
|
|
|
|
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
|
if !errors.Is(err, ErrLibraryResumeBusy) {
|
|
t.Fatalf("want ErrLibraryResumeBusy, got %v (resumed=%v)", err, resumed)
|
|
}
|
|
var state, stage string
|
|
if err := pool.QueryRow(context.Background(),
|
|
`SELECT state, stage FROM library_jobs WHERE id=$1`, jobID).Scan(&state, &stage); err != nil {
|
|
t.Fatalf("reload: %v", err)
|
|
}
|
|
if state != "failed" || stage != "resources" {
|
|
t.Fatalf("busy resume mutated job: state=%s stage=%s", state, stage)
|
|
}
|
|
var qState string
|
|
if err := pool.QueryRow(context.Background(), `SELECT state FROM jobs WHERE id=$1`, queueID).Scan(&qState); err != nil {
|
|
t.Fatalf("queue: %v", err)
|
|
}
|
|
if qState != "failed" {
|
|
t.Fatalf("busy resume mutated queue row: %s", qState)
|
|
}
|
|
}
|
|
|
|
// 非失败/取消状态不得被续作改写(并发接管后的保护)。
|
|
func TestR25ResumeRejectsNonFailedStateWithoutMutation(t *testing.T) {
|
|
store, pool := newIdentityStore(t)
|
|
jobID := "librestore-internal-r25-succeeded"
|
|
seedResumableJob(t, pool, jobID, "finalize", "succeeded", ptrTime(time.Now().UTC().Add(-time.Minute)))
|
|
|
|
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
|
if err != nil {
|
|
t.Fatalf("resume on succeeded job: %v", err)
|
|
}
|
|
if resumed {
|
|
t.Fatal("succeeded job must not be resumed")
|
|
}
|
|
var state string
|
|
if err := pool.QueryRow(context.Background(), `SELECT state FROM library_jobs WHERE id=$1`, jobID).Scan(&state); err != nil {
|
|
t.Fatalf("reload: %v", err)
|
|
}
|
|
if state != "succeeded" {
|
|
t.Fatalf("state changed on rejected resume: %s", state)
|
|
}
|
|
}
|
|
|
|
// 入队真的失败(数据库层拒绝写队列)时,attempt / job / queue 必须全部回到原值:
|
|
// 不能留下"state=pending 却没有队列行"的假 active。
|
|
func TestR26ResumeEnqueueFailureRollsBackEveryField(t *testing.T) {
|
|
store, pool := newIdentityStore(t)
|
|
ctx := context.Background()
|
|
jobID := "librestore-internal-r26-rollback-all"
|
|
past := time.Now().UTC().Add(-time.Minute)
|
|
queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &past)
|
|
if _, err := pool.Exec(ctx, `UPDATE library_jobs SET error='boom', recovery='rolled back',
|
|
lease_owner='old', lease_until=$2, attempt_epoch=3 WHERE id=$1`, jobID, past.Add(30*time.Second)); err != nil {
|
|
t.Fatalf("seed fields: %v", err)
|
|
}
|
|
before := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID)
|
|
queueBefore := rowJSON(t, pool, `SELECT to_jsonb(j) FROM jobs j WHERE id=$1`, queueID)
|
|
if _, err := pool.Exec(ctx, `CREATE FUNCTION r26_reject_queue() RETURNS trigger AS $$ BEGIN
|
|
RAISE EXCEPTION 'r26 injected queue write failure'; END; $$ LANGUAGE plpgsql;
|
|
CREATE TRIGGER r26_reject BEFORE INSERT OR UPDATE ON jobs FOR EACH ROW EXECUTE FUNCTION r26_reject_queue();`); err != nil {
|
|
t.Fatalf("install trigger: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
_, _ = pool.Exec(context.Background(), `DROP TRIGGER IF EXISTS r26_reject ON jobs;
|
|
DROP FUNCTION IF EXISTS r26_reject_queue();`)
|
|
})
|
|
if _, resumed, err := store.ResumeLibraryJob(ctx, jobID, "library.restore", time.Now().UTC()); err == nil || resumed {
|
|
t.Fatalf("resume must fail when queue write is rejected: resumed=%v err=%v", resumed, err)
|
|
}
|
|
if after := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID); after != before {
|
|
t.Fatalf("library_jobs changed after failed enqueue:\n before=%s\n after=%s", before, after)
|
|
}
|
|
if after := rowJSON(t, pool, `SELECT to_jsonb(j) FROM jobs j WHERE id=$1`, queueID); after != queueBefore {
|
|
t.Fatalf("queue row changed after failed enqueue:\n before=%s\n after=%s", queueBefore, after)
|
|
}
|
|
}
|
|
|
|
// 回滚阶段的作业必须在事务内被拒绝(不是靠 handler 读到的旧快照)。
|
|
func TestR26ResumeRejectsRollbackStageInsideTransaction(t *testing.T) {
|
|
store, pool := newIdentityStore(t)
|
|
jobID := "librestore-internal-r26-rollback-stage"
|
|
seedResumableJob(t, pool, jobID, "rollback", "failed", ptrTime(time.Now().UTC().Add(-time.Minute)))
|
|
before := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID)
|
|
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
|
if !errors.Is(err, ErrLibraryResumeRequiresRecover) || resumed {
|
|
t.Fatalf("want ErrLibraryResumeRequiresRecover, got resumed=%v err=%v", resumed, err)
|
|
}
|
|
if after := rowJSON(t, pool, `SELECT to_jsonb(j) FROM library_jobs j WHERE id=$1`, jobID); after != before {
|
|
t.Fatalf("rollback rejection mutated job:\n before=%s\n after=%s", before, after)
|
|
}
|
|
}
|
|
|
|
// 并发重复续作只允许一个成功,队列只有一条投递(不能双投)。
|
|
func TestR26ResumeConcurrentDuplicateDeliversOnce(t *testing.T) {
|
|
store, pool := newIdentityStore(t)
|
|
jobID := "librestore-internal-r26-concurrent"
|
|
past := time.Now().UTC().Add(-time.Minute)
|
|
queueID := seedResumableJob(t, pool, jobID, "resources", "failed", &past)
|
|
|
|
type res struct {
|
|
resumed bool
|
|
err error
|
|
}
|
|
ch := make(chan res, 2)
|
|
start := make(chan struct{})
|
|
for i := 0; i < 2; i++ {
|
|
go func() {
|
|
<-start
|
|
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, "library.restore", time.Now().UTC())
|
|
ch <- res{resumed, err}
|
|
}()
|
|
}
|
|
close(start)
|
|
ok, busy := 0, 0
|
|
for i := 0; i < 2; i++ {
|
|
r := <-ch
|
|
switch {
|
|
case r.resumed && r.err == nil:
|
|
ok++
|
|
case errors.Is(r.err, ErrLibraryResumeBusy) || (r.err == nil && !r.resumed):
|
|
busy++
|
|
case r.err != nil:
|
|
t.Fatalf("unexpected error: %v", r.err)
|
|
}
|
|
}
|
|
if ok != 1 || busy != 1 {
|
|
t.Fatalf("want exactly one resume and one refusal, got ok=%d refused=%d", ok, busy)
|
|
}
|
|
var state string
|
|
var attempt int
|
|
if err := pool.QueryRow(context.Background(), `SELECT state, attempt FROM jobs WHERE id=$1`, queueID).Scan(&state, &attempt); err != nil {
|
|
t.Fatalf("queue: %v", err)
|
|
}
|
|
if state != "pending" || attempt != 0 {
|
|
t.Fatalf("queue not reset exactly once: state=%s attempt=%d", state, attempt)
|
|
}
|
|
var pending int
|
|
if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM library_jobs WHERE id=$1 AND state='pending'`, jobID).Scan(&pending); err != nil {
|
|
t.Fatalf("job: %v", err)
|
|
}
|
|
if pending != 1 {
|
|
t.Fatalf("job not pending: %d", pending)
|
|
}
|
|
}
|
|
|
|
// 载荷必须来自事务内锁定行:finalize 阶段续作要带 finalize_only,而不是调用方旧快照。
|
|
func TestR26ResumePayloadComesFromLockedRow(t *testing.T) {
|
|
store, pool := newIdentityStore(t)
|
|
ctx := context.Background()
|
|
jobID := "librestore-internal-r26-finalize"
|
|
past := time.Now().UTC().Add(-time.Minute)
|
|
queueID := seedResumableJob(t, pool, jobID, "finalize", "failed", &past)
|
|
if _, err := pool.Exec(ctx, `UPDATE library_jobs SET snapshot_id='snap-written' WHERE id=$1`, jobID); err != nil {
|
|
t.Fatalf("seed snapshot: %v", err)
|
|
}
|
|
if _, resumed, err := store.ResumeLibraryJob(ctx, jobID, "library.restore", time.Now().UTC()); err != nil || !resumed {
|
|
t.Fatalf("resume: resumed=%v err=%v", resumed, err)
|
|
}
|
|
var payload []byte
|
|
if err := pool.QueryRow(ctx, `SELECT payload FROM jobs WHERE id=$1`, queueID).Scan(&payload); err != nil {
|
|
t.Fatalf("payload: %v", err)
|
|
}
|
|
var decoded map[string]any
|
|
if err := json.Unmarshal(payload, &decoded); err != nil {
|
|
t.Fatalf("payload json: %v", err)
|
|
}
|
|
if decoded["finalize_only"] != true || decoded["target_snapshot_id"] != "snap-target" {
|
|
t.Fatalf("payload not built from locked row: %v", decoded)
|
|
}
|
|
}
|
|
|
|
// rowJSON 取整行快照(列顺序无关),用于"逐字段前后完全一致"的断言。
|
|
func rowJSON(t *testing.T, pool *pgxpool.Pool, query, id string) string {
|
|
t.Helper()
|
|
var s string
|
|
if err := pool.QueryRow(context.Background(), query, id).Scan(&s); err != nil {
|
|
t.Fatalf("row json: %v", err)
|
|
}
|
|
return s
|
|
}
|
|
|
|
func ptrTime(t time.Time) *time.Time { return &t }
|