This commit is contained in:
@@ -1,5 +1,19 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照(续作失败把作业留成假 active)
|
||||
|
||||
### Fixed
|
||||
|
||||
- 作业续作(`POST /api/v1/library/jobs/{id}/resume`)不再先提交 `state=pending` 再单独入队。
|
||||
旧实现里队列行主键由业务作业 id 派生(`QueueJobIDFor`),第二次续作时 `INSERT` 撞主键,
|
||||
入队失败返回 500,而作业已经被改成 `pending` 且没有可用队列行 —— 从此永久卡住,
|
||||
后续恢复请求全部被 `409 library_restore_active` 挡住。
|
||||
现在「结束上一代 attempt + 业务作业置 pending + 队列行 upsert(含租约检查)」在同一个事务里完成:
|
||||
入队失败整笔回滚、作业状态不变;租约仍有效时返回 `library_resume_busy` 而不是半态。
|
||||
- 续作改为拒绝 `stage=rollback` 的作业(`409 library_resume_requires_recover`):这类作业已经回滚并
|
||||
保留维护窗口,正确出口是维护恢复(`/library/maintenance/recover`),它会绑定原 `window_job_id`
|
||||
并使用原 before-image;继续续作等于丢开前镜像重来。
|
||||
|
||||
## 开发快照(发布页维护文案与版本记录页加载失败)
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -4,13 +4,13 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"knowledge-governance/internal/platform/postgres"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
// 按指定整库版本恢复:与同步分开的作业类型,需要**单独的管理员授权**,
|
||||
@@ -271,9 +271,37 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
// id 落盘。另建 `-resume` 作业只能继承 operation,其它所有者仍指向旧作业:
|
||||
// AcquireInstanceMaintenance 会拒绝它、journal 与预备份也查不到——那样"续作"只是
|
||||
// 一个漂亮的新行,接不上原窗口。原作业 + 新 attempt 才是批准协议的形状。
|
||||
prepared, resumed, err := store.PrepareLibraryJobResume(r.Context(), sourceID, now)
|
||||
// 已回滚并持有维护窗口的作业不能续作:继续跑等于丢开原 before-image 重来一遍,
|
||||
// 正确出口是维护恢复(/library/maintenance/recover),它绑定原 window_job_id。
|
||||
if source.Stage == "rollback" {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_requires_recover",
|
||||
"error": "作业已在回滚阶段失败并保留维护窗口,请走维护恢复而不是续作"})
|
||||
return
|
||||
}
|
||||
// 写入已完成、只差登记版本的情况:续作只做幂等收尾,不重写生产。
|
||||
body := map[string]any{"library_job_id": sourceID, "scope": source.Scope,
|
||||
"target_snapshot_id": source.TargetSnapshotID}
|
||||
if source.Stage == "finalize" && source.SnapshotID != "" {
|
||||
body["finalize_only"] = true
|
||||
}
|
||||
payload, _ := json.Marshal(body)
|
||||
queueID := governance.QueueJobIDFor(sourceID)
|
||||
// 状态变更与队列 upsert 必须在同一事务里:否则入队失败会把作业留在 pending 却没有队列行。
|
||||
resumer, ok := config.Store.(interface {
|
||||
ResumeLibraryJob(ctx context.Context, jobID, queueID, queueType string, payload []byte, now time.Time) (governance.LibraryJob, bool, error)
|
||||
})
|
||||
if !ok {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不支持续作"})
|
||||
return
|
||||
}
|
||||
prepared, resumed, err := resumer.ResumeLibraryJob(r.Context(), sourceID, queueID, queueType, payload, now)
|
||||
if errors.Is(err, postgres.ErrLibraryResumeBusy) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_busy",
|
||||
"error": "该作业已有执行在跑(租约仍有效),等它结束再续作"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot prepare resumed job"})
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot resume job"})
|
||||
return
|
||||
}
|
||||
if !resumed {
|
||||
@@ -282,31 +310,6 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
"error": "作业已不在可续作状态(可能已被接管或完成)"})
|
||||
return
|
||||
}
|
||||
// 写入已完成、只差登记版本的情况:续作只做幂等收尾,不重写生产。
|
||||
body := map[string]any{"library_job_id": prepared.ID, "scope": source.Scope,
|
||||
"target_snapshot_id": source.TargetSnapshotID}
|
||||
if source.Stage == "finalize" && source.SnapshotID != "" {
|
||||
body["finalize_only"] = true
|
||||
}
|
||||
payload, _ := json.Marshal(body)
|
||||
queueID := governance.QueueJobIDFor(prepared.ID)
|
||||
if _, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||||
ID: queueID, Type: queueType, Payload: payload, AvailableAt: now,
|
||||
MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + prepared.ID,
|
||||
}, now); err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue resumed job"})
|
||||
return
|
||||
}
|
||||
// 队列行可能还是上一次的终态:重置为 pending。仍有效的租约 ⇒ 有执行在跑,让路。
|
||||
if resetErr := store.ResetQueueJobForResume(r.Context(), queueID, now); resetErr != nil {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_busy",
|
||||
"error": "该作业已有执行在跑(租约仍有效),等它结束再续作", "detail": errString(resetErr)})
|
||||
return
|
||||
}
|
||||
if attachErr := store.AttachLibraryQueueJob(r.Context(), prepared.ID, queueID, now); attachErr != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach resumed job"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": prepared, "resumed_from": sourceID,
|
||||
"resumed": true, "next_attempt_epoch": prepared.AttemptEpoch + 1})
|
||||
}
|
||||
|
||||
@@ -502,6 +502,67 @@ WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC())
|
||||
return job, tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
// ErrLibraryResumeBusy 表示队列里那一代仍持有有效租约,续作必须让路。
|
||||
var ErrLibraryResumeBusy = errors.New("library resume busy")
|
||||
|
||||
// ResumeLibraryJob 在同一事务里完成续作所需的全部持久变更:
|
||||
// 结束上一代 attempt、把业务作业置回 pending、把队列行 upsert 成 pending。
|
||||
//
|
||||
// 为什么必须放进一个事务:旧实现先 PrepareLibraryJobResume 提交 state='pending',再由 API 单独
|
||||
// Enqueue。队列行的主键由业务作业 id 派生(QueueJobIDFor),第二次续作时 INSERT 会撞主键——
|
||||
// 入队失败,但 state 已经变成 pending 且没有队列行,作业就永久卡在"看起来在跑"的假 active 上,
|
||||
// 后续恢复全部被 409 挡住。状态与队列必须同生共死。
|
||||
func (s *Store) ResumeLibraryJob(ctx context.Context, jobID, queueID, queueType string, payload []byte, now time.Time) (governance.LibraryJob, bool, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("resume library job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_attempts SET state='abandoned', finished_at=$2, outcome='superseded by operator resume',
|
||||
lease_until=NULL
|
||||
WHERE job_id=$1 AND state IN ('claimed','running')`, jobID, now.UTC()); err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("retire attempts for resume: %w", err)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE library_jobs SET state='pending', error='', recovery='', finished_at=NULL, updated_at=$2,
|
||||
lease_owner='', lease_until=NULL, queue_job_id=$3
|
||||
WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC(), queueID)
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return governance.LibraryJob{}, false, nil
|
||||
}
|
||||
// 队列行可能已存在(上一次执行)。先锁住它判断租约:有活跃执行者就整笔回滚,绝不留半态。
|
||||
var lease *time.Time
|
||||
row := tx.QueryRow(ctx, `SELECT lease_until FROM jobs WHERE id=$1 FOR UPDATE`, queueID)
|
||||
if err := row.Scan(&lease); err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("inspect queue job for resume: %w", err)
|
||||
}
|
||||
if lease != nil && lease.After(now.UTC()) {
|
||||
return governance.LibraryJob{}, false, ErrLibraryResumeBusy
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO jobs (id, type, payload, state, priority, available_at, max_attempts, idempotency_key, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $5, $5)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
type=EXCLUDED.type, payload=EXCLUDED.payload, state='pending', attempt=0,
|
||||
lease_owner=NULL, lease_until=NULL, available_at=EXCLUDED.available_at,
|
||||
max_attempts=EXCLUDED.max_attempts, updated_at=EXCLUDED.updated_at`,
|
||||
queueID, queueType, payload, 0, now.UTC(), 1, "library.resume.queue:"+jobID); err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("enqueue resumed job: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("resume library job: %w", err)
|
||||
}
|
||||
job, loadErr := s.LibraryJob(ctx, jobID)
|
||||
if loadErr != nil {
|
||||
return governance.LibraryJob{}, false, loadErr
|
||||
}
|
||||
return job, true, nil
|
||||
}
|
||||
|
||||
// ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零,
|
||||
// 否则 attempt<max_attempts 永远不成立,作业再也不会被任何 worker 领取。
|
||||
//
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// seedResumableJob 建一条 failed/rollback 的库作业,并按 QueueJobIDFor 预置"上一次执行"留下的队列行,
|
||||
// 复现线上真实形态:队列行主键已存在、idempotency_key 与续作不同(因此旧实现 INSERT 撞主键)。
|
||||
func seedResumableJob(t *testing.T, pool *pgxpool.Pool, jobID, stage, state string, queueLease *time.Time) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO library_jobs (id, scope, kind, state, stage, idempotency_key, request_digest, request,
|
||||
queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, created_at, updated_at, operation_id)
|
||||
VALUES ($1::text,'internal','restore',$2::text,$3::text,'idem-'||$1::text,'digest-'||$1::text,'{}'::jsonb,
|
||||
$4::text,'snap-before','snap-target','tester','boom','', $5::timestamptz, $5::timestamptz, 'op-'||$1::text)`,
|
||||
jobID, state, stage, governance.QueueJobIDFor(jobID), now); err != nil {
|
||||
t.Fatalf("seed library job: %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO jobs (id, type, payload, state, priority, available_at, max_attempts, idempotency_key,
|
||||
created_at, updated_at, lease_owner, lease_until, attempt)
|
||||
VALUES ($1::text,'library.restore','{}'::jsonb,'failed',0,$2::timestamptz,1,'library.restore.queue:'||$3::text,$2::timestamptz,$2::timestamptz,'old-worker',$4::timestamptz,1)`,
|
||||
governance.QueueJobIDFor(jobID), now, jobID, queueLease); err != nil {
|
||||
t.Fatalf("seed queue job: %v", err)
|
||||
}
|
||||
return governance.QueueJobIDFor(jobID)
|
||||
}
|
||||
|
||||
// RED 的真实形态:队列行已存在时,续作必须成功复用该行;旧实现会在 Enqueue 撞主键后
|
||||
// 留下 state=pending 且没有可用队列行的假 active。
|
||||
func TestR25ResumeReusesExistingQueueRowAtomically(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
jobID := "librestore-internal-r25-atomic"
|
||||
past := time.Now().UTC().Add(-time.Minute)
|
||||
queueID := seedResumableJob(t, pool, jobID, "rollback", "failed", &past)
|
||||
|
||||
job, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, queueID, "library.restore", []byte(`{}`), time.Now().UTC())
|
||||
if err != nil {
|
||||
t.Fatalf("resume with existing queue row: %v", err)
|
||||
}
|
||||
if !resumed || job.State != governance.LibraryJobPending {
|
||||
t.Fatalf("resume did not take effect: resumed=%v state=%s", resumed, job.State)
|
||||
}
|
||||
row := pool.QueryRow(context.Background(),
|
||||
`SELECT state, attempt, coalesce(lease_owner,'') FROM jobs WHERE id=$1`, queueID)
|
||||
var state, owner string
|
||||
var attempt int
|
||||
if err := row.Scan(&state, &attempt, &owner); err != nil {
|
||||
t.Fatalf("queue row missing after resume: %v", err)
|
||||
}
|
||||
if state != "pending" || attempt != 0 || owner != "" {
|
||||
t.Fatalf("queue row not reset for resume: state=%s attempt=%d owner=%q", state, attempt, owner)
|
||||
}
|
||||
if job.QueueJobID != queueID {
|
||||
t.Fatalf("library job not attached to queue row: %q", job.QueueJobID)
|
||||
}
|
||||
}
|
||||
|
||||
// 有有效租约时必须整笔回滚:不能出现"业务作业 pending、队列仍被别人持有"的半态。
|
||||
func TestR25ResumeBusyLeaseDoesNotMutateState(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
jobID := "librestore-internal-r25-busy"
|
||||
future := time.Now().UTC().Add(10 * time.Minute)
|
||||
queueID := seedResumableJob(t, pool, jobID, "rollback", "failed", &future)
|
||||
|
||||
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, queueID, "library.restore", []byte(`{}`), time.Now().UTC())
|
||||
if !errors.Is(err, ErrLibraryResumeBusy) {
|
||||
t.Fatalf("want ErrLibraryResumeBusy, got %v (resumed=%v)", err, resumed)
|
||||
}
|
||||
var state, stage string
|
||||
if err := pool.QueryRow(context.Background(),
|
||||
`SELECT state, stage FROM library_jobs WHERE id=$1`, jobID).Scan(&state, &stage); err != nil {
|
||||
t.Fatalf("reload: %v", err)
|
||||
}
|
||||
if state != "failed" || stage != "rollback" {
|
||||
t.Fatalf("busy resume mutated job: state=%s stage=%s", state, stage)
|
||||
}
|
||||
var qState string
|
||||
if err := pool.QueryRow(context.Background(), `SELECT state FROM jobs WHERE id=$1`, queueID).Scan(&qState); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
if qState != "failed" {
|
||||
t.Fatalf("busy resume mutated queue row: %s", qState)
|
||||
}
|
||||
}
|
||||
|
||||
// 非失败/取消状态不得被续作改写(并发接管后的保护)。
|
||||
func TestR25ResumeRejectsNonFailedStateWithoutMutation(t *testing.T) {
|
||||
store, pool := newIdentityStore(t)
|
||||
jobID := "librestore-internal-r25-succeeded"
|
||||
past := time.Now().UTC().Add(-time.Minute)
|
||||
queueID := seedResumableJob(t, pool, jobID, "finalize", "succeeded", &past)
|
||||
|
||||
_, resumed, err := store.ResumeLibraryJob(context.Background(), jobID, queueID, "library.restore", []byte(`{}`), time.Now().UTC())
|
||||
if err != nil {
|
||||
t.Fatalf("resume on succeeded job: %v", err)
|
||||
}
|
||||
if resumed {
|
||||
t.Fatal("succeeded job must not be resumed")
|
||||
}
|
||||
var state string
|
||||
if err := pool.QueryRow(context.Background(), `SELECT state FROM library_jobs WHERE id=$1`, jobID).Scan(&state); err != nil {
|
||||
t.Fatalf("reload: %v", err)
|
||||
}
|
||||
if state != "succeeded" {
|
||||
t.Fatalf("state changed on rejected resume: %s", state)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user