From e292b6f674bcefe893c6f295190c68ec5d20b5c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Sun, 1 Nov 2026 21:51:43 +0800 Subject: [PATCH] =?UTF-8?q?fix(library):=20=E7=BB=AD=E4=BD=9C=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E5=8D=95=E4=BA=8B=E5=8A=A1=E5=85=A5=E9=98=9F=EF=BC=8C?= =?UTF-8?q?=E5=9B=9E=E6=BB=9A=E4=BD=9C=E4=B8=9A=E8=BD=AC=E7=BB=B4=E6=8A=A4?= =?UTF-8?q?=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 14 +++ internal/httpapi/library_restore.go | 59 ++++----- internal/platform/postgres/library_jobs.go | 61 +++++++++ .../postgres/library_resume_r25_test.go | 117 ++++++++++++++++++ 4 files changed, 223 insertions(+), 28 deletions(-) create mode 100644 internal/platform/postgres/library_resume_r25_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fb3024..9b1c685 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 开发快照(续作失败把作业留成假 active) + +### Fixed + +- 作业续作(`POST /api/v1/library/jobs/{id}/resume`)不再先提交 `state=pending` 再单独入队。 + 旧实现里队列行主键由业务作业 id 派生(`QueueJobIDFor`),第二次续作时 `INSERT` 撞主键, + 入队失败返回 500,而作业已经被改成 `pending` 且没有可用队列行 —— 从此永久卡住, + 后续恢复请求全部被 `409 library_restore_active` 挡住。 + 现在「结束上一代 attempt + 业务作业置 pending + 队列行 upsert(含租约检查)」在同一个事务里完成: + 入队失败整笔回滚、作业状态不变;租约仍有效时返回 `library_resume_busy` 而不是半态。 +- 续作改为拒绝 `stage=rollback` 的作业(`409 library_resume_requires_recover`):这类作业已经回滚并 + 保留维护窗口,正确出口是维护恢复(`/library/maintenance/recover`),它会绑定原 `window_job_id` + 并使用原 before-image;继续续作等于丢开前镜像重来。 + ## 开发快照(发布页维护文案与版本记录页加载失败) ### Fixed diff --git a/internal/httpapi/library_restore.go b/internal/httpapi/library_restore.go index d1eca63..9ccf76c 100644 --- a/internal/httpapi/library_restore.go +++ b/internal/httpapi/library_restore.go @@ -4,13 +4,13 @@ import ( "context" "encoding/json" "errors" + "knowledge-governance/internal/platform/postgres" "net/http" "strconv" "strings" "time" "knowledge-governance/internal/governance" - "knowledge-governance/internal/jobs" ) // 按指定整库版本恢复:与同步分开的作业类型,需要**单独的管理员授权**, @@ -271,9 +271,37 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) { // id 落盘。另建 `-resume` 作业只能继承 operation,其它所有者仍指向旧作业: // AcquireInstanceMaintenance 会拒绝它、journal 与预备份也查不到——那样"续作"只是 // 一个漂亮的新行,接不上原窗口。原作业 + 新 attempt 才是批准协议的形状。 - prepared, resumed, err := store.PrepareLibraryJobResume(r.Context(), sourceID, now) + // 已回滚并持有维护窗口的作业不能续作:继续跑等于丢开原 before-image 重来一遍, + // 正确出口是维护恢复(/library/maintenance/recover),它绑定原 window_job_id。 + if source.Stage == "rollback" { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_requires_recover", + "error": "作业已在回滚阶段失败并保留维护窗口,请走维护恢复而不是续作"}) + return + } + // 写入已完成、只差登记版本的情况:续作只做幂等收尾,不重写生产。 + body := map[string]any{"library_job_id": sourceID, "scope": source.Scope, + "target_snapshot_id": source.TargetSnapshotID} + if source.Stage == "finalize" && source.SnapshotID != "" { + body["finalize_only"] = true + } + payload, _ := json.Marshal(body) + queueID := governance.QueueJobIDFor(sourceID) + // 状态变更与队列 upsert 必须在同一事务里:否则入队失败会把作业留在 pending 却没有队列行。 + resumer, ok := config.Store.(interface { + ResumeLibraryJob(ctx context.Context, jobID, queueID, queueType string, payload []byte, now time.Time) (governance.LibraryJob, bool, error) + }) + if !ok { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不支持续作"}) + return + } + prepared, resumed, err := resumer.ResumeLibraryJob(r.Context(), sourceID, queueID, queueType, payload, now) + if errors.Is(err, postgres.ErrLibraryResumeBusy) { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_busy", + "error": "该作业已有执行在跑(租约仍有效),等它结束再续作"}) + return + } if err != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot prepare resumed job"}) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot resume job"}) return } if !resumed { @@ -282,31 +310,6 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) { "error": "作业已不在可续作状态(可能已被接管或完成)"}) return } - // 写入已完成、只差登记版本的情况:续作只做幂等收尾,不重写生产。 - body := map[string]any{"library_job_id": prepared.ID, "scope": source.Scope, - "target_snapshot_id": source.TargetSnapshotID} - if source.Stage == "finalize" && source.SnapshotID != "" { - body["finalize_only"] = true - } - payload, _ := json.Marshal(body) - queueID := governance.QueueJobIDFor(prepared.ID) - if _, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{ - ID: queueID, Type: queueType, Payload: payload, AvailableAt: now, - MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + prepared.ID, - }, now); err != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue resumed job"}) - return - } - // 队列行可能还是上一次的终态:重置为 pending。仍有效的租约 ⇒ 有执行在跑,让路。 - if resetErr := store.ResetQueueJobForResume(r.Context(), queueID, now); resetErr != nil { - writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_busy", - "error": "该作业已有执行在跑(租约仍有效),等它结束再续作", "detail": errString(resetErr)}) - return - } - if attachErr := store.AttachLibraryQueueJob(r.Context(), prepared.ID, queueID, now); attachErr != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach resumed job"}) - return - } writeJSON(w, http.StatusAccepted, map[string]any{"job": prepared, "resumed_from": sourceID, "resumed": true, "next_attempt_epoch": prepared.AttemptEpoch + 1}) } diff --git a/internal/platform/postgres/library_jobs.go b/internal/platform/postgres/library_jobs.go index cd2d7df..593179a 100644 --- a/internal/platform/postgres/library_jobs.go +++ b/internal/platform/postgres/library_jobs.go @@ -502,6 +502,67 @@ WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC()) return job, tag.RowsAffected() == 1, nil } +// ErrLibraryResumeBusy 表示队列里那一代仍持有有效租约,续作必须让路。 +var ErrLibraryResumeBusy = errors.New("library resume busy") + +// ResumeLibraryJob 在同一事务里完成续作所需的全部持久变更: +// 结束上一代 attempt、把业务作业置回 pending、把队列行 upsert 成 pending。 +// +// 为什么必须放进一个事务:旧实现先 PrepareLibraryJobResume 提交 state='pending',再由 API 单独 +// Enqueue。队列行的主键由业务作业 id 派生(QueueJobIDFor),第二次续作时 INSERT 会撞主键—— +// 入队失败,但 state 已经变成 pending 且没有队列行,作业就永久卡在"看起来在跑"的假 active 上, +// 后续恢复全部被 409 挡住。状态与队列必须同生共死。 +func (s *Store) ResumeLibraryJob(ctx context.Context, jobID, queueID, queueType string, payload []byte, now time.Time) (governance.LibraryJob, bool, error) { + tx, err := s.db.Begin(ctx) + if err != nil { + return governance.LibraryJob{}, false, fmt.Errorf("resume library job: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + if _, err := tx.Exec(ctx, ` +UPDATE library_attempts SET state='abandoned', finished_at=$2, outcome='superseded by operator resume', + lease_until=NULL +WHERE job_id=$1 AND state IN ('claimed','running')`, jobID, now.UTC()); err != nil { + return governance.LibraryJob{}, false, fmt.Errorf("retire attempts for resume: %w", err) + } + tag, err := tx.Exec(ctx, ` +UPDATE library_jobs SET state='pending', error='', recovery='', finished_at=NULL, updated_at=$2, + lease_owner='', lease_until=NULL, queue_job_id=$3 +WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC(), queueID) + if err != nil { + return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err) + } + if tag.RowsAffected() != 1 { + return governance.LibraryJob{}, false, nil + } + // 队列行可能已存在(上一次执行)。先锁住它判断租约:有活跃执行者就整笔回滚,绝不留半态。 + var lease *time.Time + row := tx.QueryRow(ctx, `SELECT lease_until FROM jobs WHERE id=$1 FOR UPDATE`, queueID) + if err := row.Scan(&lease); err != nil && !errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryJob{}, false, fmt.Errorf("inspect queue job for resume: %w", err) + } + if lease != nil && lease.After(now.UTC()) { + return governance.LibraryJob{}, false, ErrLibraryResumeBusy + } + if _, err := tx.Exec(ctx, ` +INSERT INTO jobs (id, type, payload, state, priority, available_at, max_attempts, idempotency_key, created_at, updated_at) +VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $5, $5) +ON CONFLICT (id) DO UPDATE SET + type=EXCLUDED.type, payload=EXCLUDED.payload, state='pending', attempt=0, + lease_owner=NULL, lease_until=NULL, available_at=EXCLUDED.available_at, + max_attempts=EXCLUDED.max_attempts, updated_at=EXCLUDED.updated_at`, + queueID, queueType, payload, 0, now.UTC(), 1, "library.resume.queue:"+jobID); err != nil { + return governance.LibraryJob{}, false, fmt.Errorf("enqueue resumed job: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return governance.LibraryJob{}, false, fmt.Errorf("resume library job: %w", err) + } + job, loadErr := s.LibraryJob(ctx, jobID) + if loadErr != nil { + return governance.LibraryJob{}, false, loadErr + } + return job, true, nil +} + // ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零, // 否则 attempt