Files
knowledge-governance/internal/platform/postgres/library_jobs.go
T
姚章浩 7ab2eb1ccf
build / build (push) Successful in 1m47s
fix(library): 按目标身份隔离版本授权与执行历史
2026-10-31 02:15:15 +08:00

1080 lines
50 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package postgres
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
// 整库作业与版本:队列(jobs)负责租约重试,本文件负责业务身份、阶段与结果。
// 互斥与幂等都由数据库唯一索引保证,不依赖前端或单 worker 部署。
func (s *Store) CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error) {
request := job.RequestDigest
if request == "" {
return governance.LibraryJob{}, false, errors.New("library job requires a request digest")
}
rawRequest, err := json.Marshal(map[string]any{"digest": request, "kind": string(job.Kind), "scope": job.Scope})
if err != nil {
return governance.LibraryJob{}, false, err
}
var id string
err = s.db.QueryRow(ctx, `
INSERT INTO library_jobs (id, scope, kind, state, stage, idempotency_key, request_digest, request,
queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, operation_id, created_at, updated_at, target_id)
VALUES ($1,$2,$3,'pending','',$4,$5,$6,$7,$8,$9,$10,'','',$11,$12,$12,$13)
ON CONFLICT (idempotency_key) DO NOTHING
RETURNING id`,
job.ID, job.Scope, string(job.Kind), job.IdempotencyKey, request, rawRequest,
job.QueueJobID, job.SnapshotID, job.TargetSnapshotID, job.Actor, job.OperationID, now.UTC(), s.libraryTargetID).Scan(&id)
if err != nil {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
// 同一目标的写作业(sync/restore)共用一个唯一索引,跨 kind 互斥。
if pgErr.ConstraintName == "library_jobs_write_target" || pgErr.ConstraintName == "library_jobs_active" {
return governance.LibraryJob{}, false, governance.ErrLibraryJobActive
}
}
if !errors.Is(err, pgx.ErrNoRows) {
return governance.LibraryJob{}, false, fmt.Errorf("create library job: %w", err)
}
// 幂等键已存在:必须返回**已存**作业,并核对请求体是否一致。
existing, loadErr := s.libraryJobByKey(ctx, job.IdempotencyKey)
if loadErr != nil {
return governance.LibraryJob{}, false, loadErr
}
if existing.RequestDigest != request {
return existing, false, governance.ErrLibraryJobIdempotencyConflict
}
return existing, true, nil
}
for _, stage := range stages {
if _, err := s.db.Exec(ctx, `
INSERT INTO library_job_stages (job_id, stage, state, detail, updated_at)
VALUES ($1,$2,'pending','',$3)
ON CONFLICT (job_id, stage) DO NOTHING`, id, stage, now.UTC()); err != nil {
return governance.LibraryJob{}, false, fmt.Errorf("create library job stage: %w", err)
}
}
created, err := s.LibraryJob(ctx, id)
return created, false, err
}
func (s *Store) libraryJobByKey(ctx context.Context, key string) (governance.LibraryJob, error) {
var id string
if err := s.db.QueryRow(ctx, `SELECT id FROM library_jobs WHERE idempotency_key=$1`, key).Scan(&id); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return governance.LibraryJob{}, errors.New("library job idempotency key not found")
}
return governance.LibraryJob{}, fmt.Errorf("load library job by key: %w", err)
}
return s.LibraryJob(ctx, id)
}
func (s *Store) LibraryJob(ctx context.Context, id string) (governance.LibraryJob, error) {
var job governance.LibraryJob
var kind, state string
var finished *time.Time
if err := s.db.QueryRow(ctx, `
SELECT id, scope, kind, state, stage, idempotency_key, request_digest, queue_job_id, snapshot_id,
target_snapshot_id, actor, error, recovery, operation_id, attempt_epoch, created_at, updated_at, finished_at
FROM library_jobs WHERE id=$1 AND (kind='seal' OR target_id=$2)`, id, s.libraryTargetID).Scan(&job.ID, &job.Scope, &kind, &state, &job.Stage, &job.IdempotencyKey,
&job.RequestDigest, &job.QueueJobID, &job.SnapshotID, &job.TargetSnapshotID, &job.Actor, &job.Error,
&job.Recovery, &job.OperationID, &job.AttemptEpoch, &job.CreatedAt, &job.UpdatedAt, &finished); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return governance.LibraryJob{}, err
}
return governance.LibraryJob{}, fmt.Errorf("load library job: %w", err)
}
job.Kind, job.State, job.FinishedAt = governance.LibraryJobKind(kind), governance.LibraryJobState(state), finished
stages, err := s.libraryStages(ctx, id)
if err != nil {
return governance.LibraryJob{}, err
}
job.Stages = stages
return job, nil
}
// LibraryReuseSource 是一条"这个目标的持久来源"记录:成功同步过的快照,
// 或这个目标自己的预备份。资源复用只认它作来源证据。
type LibraryReuseSource struct {
JobID string
SnapshotID string
Kind string // sync | prebackup
FinishedAt time.Time
}
// LibraryReuseSources 读这个目标的持久来源(含已被恢复顶替的生效版本)。
// 同步来源要求同时存在 production_effective 版本登记:只写成功但版本没登记的
// 作业不算可审计来源。
func (s *Store) LibraryReuseSources(ctx context.Context, scope string, limit int) ([]LibraryReuseSource, error) {
if limit < 1 || limit > 200 {
limit = 100
}
rows, err := s.db.Query(ctx, `
SELECT j.id, j.target_snapshot_id, 'sync', coalesce(j.finished_at, j.updated_at)
FROM library_jobs j
WHERE j.scope=$1 AND j.target_id=$3 AND j.kind='sync' AND j.state='succeeded' AND coalesce(j.target_snapshot_id,'')<>''
AND EXISTS (SELECT 1 FROM library_versions v
WHERE v.scope=j.scope AND v.target_id=j.target_id AND v.kind='production_effective'
AND v.snapshot_id=j.target_snapshot_id AND v.source_job_id=j.id)
UNION ALL
SELECT coalesce(v.source_job_id,''), v.snapshot_id, 'prebackup', v.created_at
FROM library_versions v
WHERE v.scope=$1 AND v.target_id=$3 AND v.kind='production_prebackup' AND coalesce(v.snapshot_id,'')<>''
ORDER BY 4 DESC
LIMIT $2`, scope, limit, s.libraryTargetID)
if err != nil {
return nil, fmt.Errorf("load library reuse sources: %w", err)
}
defer rows.Close()
sources := make([]LibraryReuseSource, 0, 8)
for rows.Next() {
var source LibraryReuseSource
if err := rows.Scan(&source.JobID, &source.SnapshotID, &source.Kind, &source.FinishedAt); err != nil {
return nil, fmt.Errorf("scan library reuse source: %w", err)
}
sources = append(sources, source)
}
return sources, rows.Err()
}
func (s *Store) libraryStages(ctx context.Context, id string) ([]governance.LibraryStage, error) {
rows, err := s.db.Query(ctx, `
SELECT stage, state, total, completed, bytes_total, bytes_done, detail, updated_at
FROM library_job_stages WHERE job_id=$1 ORDER BY updated_at, stage`, id)
if err != nil {
return nil, fmt.Errorf("load library job stages: %w", err)
}
defer rows.Close()
result := make([]governance.LibraryStage, 0, 7)
for rows.Next() {
var stage governance.LibraryStage
if err := rows.Scan(&stage.Stage, &stage.State, &stage.Total, &stage.Completed, &stage.BytesTotal,
&stage.BytesDone, &stage.Detail, &stage.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan library job stage: %w", err)
}
result = append(result, stage)
}
return result, rows.Err()
}
// ListLibraryJobs 返回某目标最近的作业(按创建时间倒序),供页面在重启后恢复展示。
func (s *Store) ListLibraryJobs(ctx context.Context, scope string, limit int) ([]governance.LibraryJob, error) {
if limit < 1 || limit > 50 {
return nil, errors.New("library job list limit out of range")
}
rows, err := s.db.Query(ctx, `SELECT id FROM library_jobs WHERE scope=$1 AND (kind='seal' OR target_id=$3) ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit, s.libraryTargetID)
if err != nil {
return nil, fmt.Errorf("list library jobs: %w", err)
}
defer rows.Close()
ids := make([]string, 0, limit)
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, fmt.Errorf("scan library job id: %w", err)
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
result := make([]governance.LibraryJob, 0, len(ids))
for _, id := range ids {
job, err := s.LibraryJob(ctx, id)
if err != nil {
return nil, err
}
result = append(result, job)
}
return result, nil
}
// UpdateLibraryStage 只改调用方给了值的维度:nil 表示"这一维还没有可靠数据",
// 保持 NULL,绝不写成 0。
func (s *Store) UpdateLibraryStage(ctx context.Context, jobID, stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error {
tag, err := s.db.Exec(ctx, `
UPDATE library_job_stages SET
state = CASE WHEN $3='' THEN state ELSE $3 END,
total = COALESCE($4, total),
completed = COALESCE($5, completed),
bytes_total = COALESCE($6, bytes_total),
bytes_done = COALESCE($7, bytes_done),
detail = CASE WHEN $8='' THEN detail ELSE $8 END,
updated_at = $9
WHERE job_id=$1 AND stage=$2`, jobID, stage, state, total, completed, bytesTotal, bytesDone, detail, now.UTC())
if err != nil {
return fmt.Errorf("update library stage: %w", err)
}
if tag.RowsAffected() != 1 {
return errors.New("library stage not found")
}
return nil
}
// AttachLibraryQueueJob 把队列作业号写到已建的业务作业上(两步创建,避免先入队后才发现互斥冲突)。
func (s *Store) AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET queue_job_id=$2, updated_at=$3 WHERE id=$1`, jobID, queueJobID, now.UTC())
if err != nil {
return fmt.Errorf("attach library queue job: %w", err)
}
if tag.RowsAffected() != 1 {
return errors.New("library job not found")
}
return nil
}
func (s *Store) CancelLibraryJob(ctx context.Context, jobID, reason string, now time.Time) error {
return s.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobCancelled, "", reason, "", "", "", now)
}
func (s *Store) AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error {
finished := any(nil)
if state == governance.LibraryJobSucceeded || state == governance.LibraryJobFailed || state == governance.LibraryJobCancelled {
finished = now.UTC()
}
tag, err := s.db.Exec(ctx, `
UPDATE library_jobs SET
state=$2,
stage = CASE WHEN $3='' THEN stage ELSE $3 END,
error = CASE WHEN $4='' THEN error ELSE $4 END,
recovery = CASE WHEN $5='' THEN recovery ELSE $5 END,
snapshot_id = CASE WHEN $6='' THEN snapshot_id ELSE $6 END,
target_snapshot_id = CASE WHEN $7='' THEN target_snapshot_id ELSE $7 END,
updated_at=$8,
finished_at=COALESCE($9, finished_at)
WHERE id=$1`, jobID, string(state), stage, message, recovery, snapshotID, targetSnapshotID, now.UTC(), finished)
if err != nil {
return fmt.Errorf("advance library job: %w", err)
}
if tag.RowsAffected() != 1 {
return errors.New("library job not found")
}
return nil
}
// RecordLibraryVersion 写入一条**不可变版本事件**,并在需要时推进 current 指针。
//
// 与旧实现的区别(F02/F09 的根因):
// - 事件 ID 含 operation:不同 operation 引用同一快照是两条事件,`source_job_id`/
// `evidence`/`created_at` 谁也不覆盖谁(旧实现是 `ON CONFLICT DO UPDATE` 整份替换)。
// - 同一 operation 重放是幂等:`ON CONFLICT (id) DO NOTHING`,已存行原样返回。
// - "谁当前生效"由 `library_current_versions` 表达,用单调次序做条件更新;
// 更早登记的事实不能把指针拉回历史,旧 epoch 在写边界就被守卫拒绝(见 guard)。
// - 生效/恢复事件才动指针;预备份/封存只追加事件。
func (s *Store) RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error {
targetID := s.libraryTargetID
if version.Kind == governance.LibraryVersionTestSealed {
targetID = ""
}
evidence, err := json.Marshal(version.Evidence)
if err != nil {
return fmt.Errorf("encode library version evidence: %w", err)
}
tx, err := s.db.Begin(ctx)
if err != nil {
return fmt.Errorf("begin library version: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
// 执行代次守卫:写事件的人必须还是当前代次(失租旧执行器不能登记"我成功了")。
if version.SourceJobID != "" && version.AttemptEpoch > 0 {
if err := guardLibraryJobEpoch(ctx, tx, version.SourceJobID, version.AttemptEpoch, version.AttemptOwner); err != nil {
return err
}
}
// 事件身份 = (kind, scope, 快照, operation)。同一身份的重复登记只有两种合法形态:
// · 同一执行者补登记**新增**的字段(预备份先登记、写集合随后补上);
// · 完全没有新字段的纯重放。
// 所以冲突时只做**加法**:已有键永远赢(右操作数是已有 evidence),身份字段
// (source_job_id/operation_id/created_at)一格都不许改;换了执行者或换了意图的
// 登记在这里匹配不到行,什么也不写(F02/F09 不能靠 upsert 改写归属)。
// xmax = 0 才是真的新插入:重放同一份证据(或换了执行者被 WHERE 挡住)都不算新事实,
// 否则一次纯重放就能把 current 再"推进"一次,把后来的操作顶掉。
var inserted bool
err = tx.QueryRow(ctx, `
INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id,
operation_id, attempt_epoch, evidence, created_by, created_at, target_id)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)
ON CONFLICT (id) DO UPDATE
SET evidence = EXCLUDED.evidence || library_versions.evidence
WHERE library_versions.source_job_id = EXCLUDED.source_job_id
AND library_versions.operation_id = EXCLUDED.operation_id
AND library_versions.target_id = EXCLUDED.target_id
RETURNING (xmax = 0)`,
version.ID, version.Scope, string(version.Kind), version.SnapshotID, version.VersionLabel,
version.SourceJobID, version.OperationID, version.AttemptEpoch, evidence, version.CreatedBy,
version.CreatedAt.UTC(), targetID).Scan(&inserted)
if errors.Is(err, pgx.ErrNoRows) {
var existingTarget string
if loadErr := tx.QueryRow(ctx, `SELECT target_id FROM library_versions WHERE id=$1`, version.ID).Scan(&existingTarget); loadErr != nil {
return loadErr
}
if existingTarget != targetID {
return errors.New("library version identity belongs to another target")
}
// 同一身份已经存在、但归属是别的执行者/别的意图:不覆盖、不报错,也不推进 current。
inserted = false
err = nil
}
if err != nil {
return fmt.Errorf("record library version event: %w", err)
}
if version.Kind == governance.LibraryVersionProductionEffective || version.Kind == governance.LibraryVersionRecoveryVerified {
if err := advanceLibraryCurrent(ctx, tx, version, inserted, targetID); err != nil {
return err
}
}
return tx.Commit(ctx)
}
// advanceLibraryCurrent 用单调次序条件推进 current 指针。
//
// 次序键 = (意图创建时间 | operation | epoch),**不是**事件登记时间(父的复核项):
// 拿登记时间排序,一次迟到的旧 operation 重放(例如重跑几天前的同步)就会因为"我刚登记"
// 而把 current 推回历史。意图的创建时间不会因为重放而变新,所以旧意图永远推不动
// 比它更新的意图。
//
// 恢复要不要能把 current 指回更早的**内容**?要,但它是通过**新的意图**(kind=recover,
// 创建于恢复时刻)做到的:内容更旧、意图更新,两者不矛盾。旧代次能不能写由 epoch
// 守卫回答(见 libraryFence),不由时间回答。
func advanceLibraryCurrent(ctx context.Context, tx pgx.Tx, version governance.LibraryVersion, inserted bool, targetID string) error {
orderAt := version.CreatedAt
if version.OperationID != "" {
var created *time.Time
err := tx.QueryRow(ctx, `SELECT created_at FROM library_operations WHERE id=$1`,
version.OperationID).Scan(&created)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("事件引用的 operation %s 不存在:拒绝推进 current", version.OperationID)
}
if err != nil {
return fmt.Errorf("load library operation for current ordering: %w", err)
}
if created != nil {
orderAt = *created
}
}
orderKey := governance.CurrentOrderKey(orderAt, version.OperationID, version.AttemptEpoch)
if !inserted {
// 重放:事件已经在台账里。确认指针没有被别的更晚事实覆盖,否则只确认自己的历史。
var currentID string
err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1 AND target_id=$2 FOR UPDATE`,
version.Scope, targetID).Scan(&currentID)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return fmt.Errorf("load library current pointer: %w", err)
}
return nil
}
tag, err := tx.Exec(ctx, `
INSERT INTO library_current_versions (scope, version_id, snapshot_id, operation_id, attempt_epoch, order_key, updated_at, target_id)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
ON CONFLICT (scope, target_id) DO UPDATE SET
version_id=EXCLUDED.version_id, snapshot_id=EXCLUDED.snapshot_id,
operation_id=EXCLUDED.operation_id, attempt_epoch=EXCLUDED.attempt_epoch,
order_key=EXCLUDED.order_key, updated_at=EXCLUDED.updated_at
WHERE EXCLUDED.order_key > library_current_versions.order_key`,
version.Scope, version.ID, version.SnapshotID, version.OperationID, version.AttemptEpoch, orderKey,
version.CreatedAt.UTC(), targetID)
if err != nil {
return fmt.Errorf("advance library current pointer: %w", err)
}
var currentID string
if err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1 AND target_id=$2`,
version.Scope, targetID).Scan(&currentID); err != nil {
return fmt.Errorf("read back library current pointer: %w", err)
}
if tag.RowsAffected() == 0 && currentID != version.ID {
return fmt.Errorf("%w(scope=%s:当前 %s,本条 %s)", governance.ErrLibraryCurrentStale,
version.Scope, currentID, version.ID)
}
// superseded_at 只是展示元数据("这条已被顶替"),不改任何证据字段。
// 它跟着指针走:指针指向谁,谁就是未顶替的那一行。
if _, err := tx.Exec(ctx, `
UPDATE library_versions SET superseded_at = CASE WHEN id=$2 THEN NULL ELSE COALESCE(superseded_at, $3) END
WHERE scope=$1 AND target_id=$4 AND kind IN ('production_effective','recovery_verified')`, version.Scope, currentID,
version.CreatedAt.UTC(), targetID); err != nil {
return fmt.Errorf("refresh library version supersede metadata: %w", err)
}
return nil
}
// InterruptedLibraryJobs 找出"业务作业 running、但队列里已没有活跃作业"的作业:
// worker 被杀/重启后它们需要被接管,而不是被当成失败丢掉已完成的预备份。
func (s *Store) InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error) {
// lease 过期才算中断:worker 刚被杀但 lease 未到期时,队列里还是 running,
// 直接判"没中断"会让作业永远悬着;所以这里按 lease_until 判断。
rows, err := s.db.Query(ctx, `
SELECT lj.id FROM library_jobs lj
LEFT JOIN jobs j ON j.id=lj.queue_job_id
WHERE lj.kind IN ('sync','restore','baseline')
AND (
-- 业务作业在跑、队列已经没了/停了/租约过期
(lj.state='running' AND (j.id IS NULL OR j.state IN ('failed','completed','cancelled','dead')
OR (j.state IN ('pending','running','retry') AND j.lease_until IS NOT NULL AND j.lease_until < now())))
-- 业务作业已被前一轮接管置为 pending,但队列作业的租约也没了(worker 又死了)
OR (lj.state='pending' AND j.state='running' AND j.lease_until IS NOT NULL AND j.lease_until < now())
)
AND lj.target_id=$1
ORDER BY lj.created_at`, s.libraryTargetID)
if err != nil {
return nil, fmt.Errorf("list interrupted library jobs: %w", err)
}
defer rows.Close()
ids := make([]string, 0, 4)
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
jobs := make([]governance.LibraryJob, 0, len(ids))
for _, id := range ids {
job, err := s.LibraryJob(ctx, id)
if err != nil {
return nil, err
}
jobs = append(jobs, job)
}
return jobs, nil
}
// PrepareLibraryJobResume 把**同一个业务作业**放回 pending,供续作/接管重新执行。
//
// 为什么是原作业而不是新开一行(R3):维护窗口 owner_job_id、freeze journal 文件名、
// 预备份与写集合的身份全部按业务作业 id 落在持久层。新开 `-resume` 作业只能继承
// operation_id,其它所有者都还指向旧作业——AcquireInstanceMaintenance 会拒绝它,
// journal/预备份也查不到,于是"续作"只是一个漂亮的新行。
//
// 条件更新:只有 failed/cancelled 才允许置回 pending;终态时间与错误清零,
// 旧 attempt 行保留(历史不可变)。代次由 BeginLibraryAttempt 在 worker 起手时递增。
func (s *Store) PrepareLibraryJobResume(ctx context.Context, jobID string, now time.Time) (governance.LibraryJob, bool, error) {
if _, err := s.LibraryJob(ctx, jobID); err != nil {
return governance.LibraryJob{}, false, err
}
tx, err := s.db.Begin(ctx)
if err != nil {
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
// 续作是人工发起的接管:作业已经 failed/cancelled,它那一代执行者不可能还在合法地写。
// 必须**显式结束**那一代(行保留、标 abandoned),否则它的租约还没过期,
// BeginLibraryAttempt 会认为"有活跃执行者"而拒绝新代次——续作就永远开不起来。
if _, err := tx.Exec(ctx, `
UPDATE library_attempts SET state='abandoned', finished_at=$2, outcome='superseded by operator resume',
lease_until=NULL
WHERE job_id=$1 AND state IN ('claimed','running')`, jobID, now.UTC()); err != nil {
return governance.LibraryJob{}, false, fmt.Errorf("retire attempts for resume: %w", err)
}
tag, err := tx.Exec(ctx, `
UPDATE library_jobs SET state='pending', error='', recovery='', finished_at=NULL, updated_at=$2,
lease_owner='', lease_until=NULL
WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC())
if err != nil {
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
}
job, loadErr := s.LibraryJob(ctx, jobID)
if loadErr != nil {
return governance.LibraryJob{}, false, loadErr
}
return job, tag.RowsAffected() == 1, nil
}
// ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零,
// 否则 attempt<max_attempts 永远不成立,作业再也不会被任何 worker 领取。
//
// 但**不能清掉仍然有效的租约**:那等于把正在执行的那一代直接踢掉,而它可能已经在写目标
// (F13 的原始形态)。只有租约为空或已过期才允许重置。
func (s *Store) ResetQueueJobForResume(ctx context.Context, queueJobID string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE jobs
SET state='pending', attempt=0, lease_owner=NULL, lease_until=NULL, available_at=$2, updated_at=$2
WHERE id=$1 AND state IN ('running','retry','failed','pending')
AND (lease_until IS NULL OR lease_until < $2)`, queueJobID, now.UTC())
if err != nil {
return fmt.Errorf("reset queue job for resume: %w", err)
}
if tag.RowsAffected() == 0 {
// 有活跃租约:另一个执行者正在跑,本次接管必须让路(调用方跳过即可)。
return ErrLibraryAttemptHeld
}
return nil
}
// ClaimLibraryJobForResume 用一次条件更新把中断作业抢过来:只有把 running 改成 pending
// 的那一个 worker 拿到 true,多个 worker 同时重启也只会有一个执行(fencing)。
//
// 条件里必须看租约:pending→pending 的无条件认领会让两个 worker 都"认领成功",
// 然后各自跑一遍同一个窗口(F13)。租约仍有效时直接失败,让调用方跳过。
func (s *Store) ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error) {
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET state='pending', updated_at=$2
WHERE id=$1 AND target_id=$3 AND state IN ('running','pending')
AND (lease_owner='' OR lease_until IS NULL OR lease_until < $2)`, jobID, now.UTC(), s.libraryTargetID)
if err != nil {
return false, fmt.Errorf("claim interrupted library job: %w", err)
}
return tag.RowsAffected() == 1, nil
}
// ReconcileLibraryJobs 收拾"业务作业还在飞、队列里已经没有对应作业"的孤儿:
// 它们会永久占用 (scope,kind) 唯一索引,让同一目标再也发不出作业。
// 只标失败并写明恢复责任,不猜结果、不重放。
//
// kinds 是**本进程负责的那几类**:两台执行器(本机封存 / 云端发布)各自收拾自己的
// 作业,跨域"顺手收拾"会把别人的在飞作业标失败,并把恢复责任写错(封存要 agent
// 恢复来源,发布要按预备份确认目标)。kinds 为空直接报错,不允许退化成"收拾全部"。
func (s *Store) ReconcileLibraryJobs(ctx context.Context, now time.Time, kinds []string) (int, error) {
if len(kinds) == 0 {
return 0, errors.New("reconcile library jobs requires the kinds this executor owns")
}
if _, err := s.db.Exec(ctx, `
UPDATE library_jobs AS lj
SET state='failed', error='队列中已无对应作业,业务作业未收到结果',
recovery=CASE WHEN lj.kind IN ('sync','restore','recover','baseline')
THEN '未确认目标状态:若已进入维护窗口,按预备份恢复并核验后再重发'
ELSE '封存作业由 agent 恢复来源;确认来源 healthy 后可重发' END,
updated_at=$1, finished_at=$1
WHERE lj.state IN ('pending','running')
AND lj.kind = ANY($2)
AND (lj.kind='seal' OR lj.target_id=$3)
AND lj.queue_job_id <> ''
AND NOT EXISTS (
SELECT 1 FROM jobs AS j
WHERE j.id = lj.queue_job_id AND j.state IN ('pending','running','retry')
)`, now.UTC(), kinds, s.libraryTargetID); err != nil {
return 0, fmt.Errorf("reconcile library jobs: %w", err)
}
// 建了业务作业但队列投递没完成(queue_job_id 为空)超过 5 分钟,也是孤儿。
tag, err := s.db.Exec(ctx, `
UPDATE library_jobs
SET state='failed', error='队列投递未完成,作业从未开始', recovery='无需恢复来源;可直接重发', updated_at=$1, finished_at=$1
WHERE state IN ('pending','running') AND queue_job_id='' AND kind = ANY($3) AND (kind='seal' OR target_id=$4) AND created_at < $2`,
now.UTC(), now.Add(-5*time.Minute).UTC(), kinds, s.libraryTargetID)
if err != nil {
return 0, fmt.Errorf("reconcile unattached library jobs: %w", err)
}
return int(tag.RowsAffected()), nil
}
// LibraryVersionByRef 按 (scope, kind, 作业, 快照) 精确定位一条版本记录。
//
// 为什么不能"取最近 N 条再在内存里筛":那是窗口,历史一旦超过窗口上限,
// 登记/接管/恢复就会把"存在"读成"不存在"——真实现场是写集合的引用无法登记,
// 报"读不到现有版本证据 / library version limit out of range"。
// 身份查询必须走索引条件,不许依赖排序窗口(把上限从 200 改成 100 只是换个数)。
//
// 同一 (scope, kind, snapshot) 只该有一行(id 由这三者决定,写入是 upsert)。
// 命中多行、或命中的那行归别的作业所有,都说明台账不唯一:返回错误让调用方
// 拒绝,不许挑一条"看起来对"的记录去恢复生产。
func (s *Store) LibraryVersionByRef(ctx context.Context, scope string, kind governance.LibraryVersionKind,
sourceJobID, snapshotID string) (governance.LibraryVersion, bool, error) {
if scope == "" || kind == "" || sourceJobID == "" || snapshotID == "" {
return governance.LibraryVersion{}, false, errors.New("library version ref requires scope, kind, job and snapshot")
}
rows, err := s.db.Query(ctx, `
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at
FROM library_versions
WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4 AND (kind='test_sealed' OR target_id=$5)
ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), sourceJobID, snapshotID, s.libraryTargetID)
if err != nil {
return governance.LibraryVersion{}, false, fmt.Errorf("load library version by ref: %w", err)
}
items, err := scanLibraryVersions(rows, 2)
if err != nil {
return governance.LibraryVersion{}, false, err
}
if len(items) > 1 {
return governance.LibraryVersion{}, false, fmt.Errorf("同一身份存在 %d 条版本记录(作业 %s,快照 %s):拒绝挑选",
len(items), sourceJobID, snapshotID)
}
if len(items) == 1 {
return items[0], true, nil
}
// 没命中时要区分"确实没有"与"这一行归别的作业":后者是身份冲突,
// 当成"没有记录"就会允许调用方覆盖别人的台账。
var owner string
err = s.db.QueryRow(ctx, `
SELECT source_job_id FROM library_versions WHERE scope=$1 AND kind=$2 AND snapshot_id=$3 AND (kind='test_sealed' OR target_id=$4)
ORDER BY created_at DESC LIMIT 1`, scope, string(kind), snapshotID, s.libraryTargetID).Scan(&owner)
if errors.Is(err, pgx.ErrNoRows) {
return governance.LibraryVersion{}, false, nil
}
if err != nil {
return governance.LibraryVersion{}, false, fmt.Errorf("load library version owner: %w", err)
}
return governance.LibraryVersion{}, false, fmt.Errorf("版本记录(快照 %s)属于作业 %s,不是 %s:拒绝借用",
snapshotID, owner, sourceJobID)
}
// scanLibraryVersions 是版本行唯一的扫描实现:两个查询(窗口列表与按身份精确查)
// 必须对同一种形状解码,否则"查得到"和"读得对"会分叉。
func scanLibraryVersions(rows pgx.Rows, capacity int) ([]governance.LibraryVersion, error) {
defer rows.Close()
result := make([]governance.LibraryVersion, 0, capacity)
for rows.Next() {
var item governance.LibraryVersion
var kind string
var evidence []byte
if err := rows.Scan(&item.ID, &item.Scope, &kind, &item.SnapshotID, &item.VersionLabel, &item.SourceJobID,
&item.OperationID, &item.AttemptEpoch, &evidence, &item.CreatedBy, &item.CreatedAt, &item.SupersededAt); err != nil {
return nil, fmt.Errorf("scan library version: %w", err)
}
item.Kind = governance.LibraryVersionKind(kind)
if len(evidence) > 0 {
_ = json.Unmarshal(evidence, &item.Evidence)
}
result = append(result, item)
}
return result, rows.Err()
}
// LibraryVersions 是**分页展示**用的窗口列表(1..100),语义是"最近 N 条"。
// 任何"当前生效 / 本快照是否生效过"的判断都不许用它:窗口一满就会把存在读成不存在。
func (s *Store) LibraryVersions(ctx context.Context, scope string, limit int) ([]governance.LibraryVersion, error) {
if limit < 1 || limit > 100 {
return nil, errors.New("library version limit out of range")
}
rows, err := s.db.Query(ctx, `
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at
FROM library_versions WHERE scope=$1 AND (kind='test_sealed' OR target_id=$3) ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit, s.libraryTargetID)
if err != nil {
return nil, fmt.Errorf("list library versions: %w", err)
}
return scanLibraryVersions(rows, limit)
}
// LibraryCurrentVersion 返回这个 scope **当前生效**的事实(由指针表回答)。
//
// 三个理由必须一起满足:
//
// · 不能用窗口列表——生效事实可能很久以前登记,之后写进来的预备份/封存会把它挤出
// 任何"最近 N 条",接管/基线/闸口就会把"已生效"读成"没有生效"。
// · 不能靠扫 library_versions.kind='production_effective' 挑未顶替行——事件不可变,
// "谁是当前"是指针的事实,不是事件行的可变字段(F02)。
// · 指针每 (scope, target_id) 至多一行,历史归属不明的空身份不归新目标。
//
// 指针可能指向 production_effective,也可能指向 recovery_verified(按预备份恢复后
// 的当前内容身份)——调用方只能用它回答"当前是什么内容",不能假设它是一次成功发布。
func (s *Store) LibraryCurrentVersion(ctx context.Context, scope string) (governance.LibraryVersion, bool, error) {
if scope == "" {
return governance.LibraryVersion{}, false, errors.New("library current version requires scope")
}
rows, err := s.db.Query(ctx, `
SELECT v.id, v.scope, v.kind, v.snapshot_id, v.version_label, v.source_job_id, v.operation_id, v.attempt_epoch,
v.evidence, v.created_by, v.created_at, v.superseded_at
FROM library_current_versions c JOIN library_versions v ON v.id = c.version_id
WHERE c.scope=$1 AND c.target_id=$2 AND v.target_id=c.target_id`, scope, s.libraryTargetID)
if err != nil {
return governance.LibraryVersion{}, false, fmt.Errorf("load current library version: %w", err)
}
items, err := scanLibraryVersions(rows, 2)
if err != nil {
return governance.LibraryVersion{}, false, err
}
if len(items) > 1 {
return governance.LibraryVersion{}, false, fmt.Errorf("scope=%s 的当前指针指向 %d 行事件:台账被写坏,拒绝挑选",
scope, len(items))
}
if len(items) == 1 {
return items[0], true, nil
}
return governance.LibraryVersion{}, false, nil
}
// LibraryVersionOfSnapshot 按 (scope, kind, 快照) 取这个快照**自己的**登记记录(不限作业)。
//
// 用途是"本快照曾经生效过吗/是否已被顶替":版本 id 由 (kind, scope, 快照) 决定,
// 一个快照在一种类型下只该有一行,作业归属会随恢复/重放被改写,所以不能用作业当条件。
// 命中多行同样拒绝挑选。
func (s *Store) LibraryVersionOfSnapshot(ctx context.Context, scope string,
kind governance.LibraryVersionKind, snapshotID string) (governance.LibraryVersion, bool, error) {
if scope == "" || kind == "" || snapshotID == "" {
return governance.LibraryVersion{}, false, errors.New("library version snapshot requires scope, kind and snapshot")
}
rows, err := s.db.Query(ctx, `
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at
FROM library_versions
WHERE scope=$1 AND kind=$2 AND snapshot_id=$3 AND (kind='test_sealed' OR target_id=$4)
ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), snapshotID, s.libraryTargetID)
if err != nil {
return governance.LibraryVersion{}, false, fmt.Errorf("load library version of snapshot: %w", err)
}
found, err := scanLibraryVersions(rows, 2)
if err != nil {
return governance.LibraryVersion{}, false, err
}
// 同一快照可以被多次意图引用,每次 operation 一条事件,这是**正常历史**;
// 但"两条同身份的行"是台账被写坏(或迁移前没有 operation 身份的未知证据),
// 这时挑选任何一条都是替调用方做决定,必须拒绝(未知证据 fail closed)。
if len(found) == 1 {
return found[0], true, nil
}
if len(found) >= 2 {
if found[0].OperationID == "" || found[1].OperationID == "" {
return governance.LibraryVersion{}, false, fmt.Errorf(
"快照 %s 在 %s 下有 %d 条无执行身份的记录:拒绝挑选", snapshotID, kind, len(found))
}
if found[0].OperationID == found[1].OperationID {
return governance.LibraryVersion{}, false, fmt.Errorf(
"快照 %s 在 %s 下同一意图登记了多行:拒绝挑选", snapshotID, kind)
}
return found[0], true, nil
}
return governance.LibraryVersion{}, false, nil
}
// CreateLibraryAuthorization 记录一次生产发布授权;同一 (scope,snapshot,digest) 只保留最新有效。
func (s *Store) CreateLibraryAuthorization(ctx context.Context, item governance.LibraryReleaseAuthorization) error {
if item.ID == "" || item.Scope == "" || item.SnapshotID == "" || item.ParamsDigest == "" || item.ExpiresAt.IsZero() {
return errors.New("invalid library release authorization")
}
_, err := s.db.Exec(ctx, `
INSERT INTO library_release_authorizations (id, scope, snapshot_id, params_digest, authorized_by, expires_at, created_at, target_id)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`,
item.ID, item.Scope, item.SnapshotID, item.ParamsDigest, item.AuthorizedBy, item.ExpiresAt.UTC(), item.CreatedAt.UTC(), s.libraryTargetID)
if err != nil {
return fmt.Errorf("create library release authorization: %w", err)
}
return nil
}
// ActiveLibraryAuthorization 只返回"未撤销、未过期、且参数摘要完全一致"的授权。
// 参数一变就查不到,自然拒绝,而不是靠调用方记得复核。
func (s *Store) ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, paramsDigest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error) {
var item governance.LibraryReleaseAuthorization
err := s.db.QueryRow(ctx, `
SELECT id, scope, snapshot_id, params_digest, authorized_by, expires_at, revoked_at, created_at
FROM library_release_authorizations
WHERE scope=$1 AND snapshot_id=$2 AND params_digest=$3 AND target_id=$5 AND revoked_at IS NULL AND expires_at > $4
ORDER BY expires_at DESC, created_at DESC
LIMIT 1`, scope, snapshotID, paramsDigest, now.UTC(), s.libraryTargetID).Scan(&item.ID, &item.Scope, &item.SnapshotID,
&item.ParamsDigest, &item.AuthorizedBy, &item.ExpiresAt, &item.RevokedAt, &item.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return governance.LibraryReleaseAuthorization{}, false, nil
}
if err != nil {
return governance.LibraryReleaseAuthorization{}, false, fmt.Errorf("load library authorization: %w", err)
}
return item, true, nil
}
func (s *Store) RevokeLibraryAuthorization(ctx context.Context, id string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_release_authorizations SET revoked_at=$2 WHERE id=$1 AND target_id=$3 AND revoked_at IS NULL`, id, now.UTC(), s.libraryTargetID)
if err != nil {
return fmt.Errorf("revoke library authorization: %w", err)
}
if tag.RowsAffected() != 1 {
return errors.New("library authorization not found or already revoked")
}
return nil
}
func (s *Store) LibraryAuthorizations(ctx context.Context, scope string, limit int) ([]governance.LibraryReleaseAuthorization, error) {
if limit < 1 || limit > 100 {
return nil, errors.New("library authorization limit out of range")
}
rows, err := s.db.Query(ctx, `
SELECT id, scope, snapshot_id, params_digest, authorized_by, expires_at, revoked_at, created_at
FROM library_release_authorizations WHERE scope=$1 AND target_id=$3 ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit, s.libraryTargetID)
if err != nil {
return nil, fmt.Errorf("list library authorizations: %w", err)
}
defer rows.Close()
result := make([]governance.LibraryReleaseAuthorization, 0, limit)
for rows.Next() {
var item governance.LibraryReleaseAuthorization
if err := rows.Scan(&item.ID, &item.Scope, &item.SnapshotID, &item.ParamsDigest, &item.AuthorizedBy,
&item.ExpiresAt, &item.RevokedAt, &item.CreatedAt); err != nil {
return nil, fmt.Errorf("scan library authorization: %w", err)
}
result = append(result, item)
}
return result, rows.Err()
}
// LibraryVersionsBySnapshot 返回这份快照在该 scope 下的全部版本事件(含证据)。
//
// 原产地不能只看 kind:目标权威基线也是 production_effective,但它是**目标侧的真实捕获**
// (证据里 action=target_baseline_init / origin=target_baseline)。所以判定必须读到证据,
// 不能把"是 effective"直接当成"来自发布"或"来自目标"。
func (s *Store) LibraryVersionsBySnapshot(ctx context.Context, scope, snapshotID string) ([]governance.LibraryVersion, error) {
if scope == "" || snapshotID == "" {
return nil, errors.New("snapshot versions requires scope and snapshot")
}
rows, err := s.db.Query(ctx, `
SELECT id, scope, kind, snapshot_id, source_job_id, COALESCE(evidence, '{}'::jsonb), created_at
FROM library_versions WHERE scope=$1 AND snapshot_id=$2 AND (kind='test_sealed' OR target_id=$3) ORDER BY created_at`, scope, snapshotID, s.libraryTargetID)
if err != nil {
return nil, fmt.Errorf("read snapshot versions: %w", err)
}
defer rows.Close()
items := make([]governance.LibraryVersion, 0, 4)
for rows.Next() {
var item governance.LibraryVersion
var kind string
var evidence []byte
if err := rows.Scan(&item.ID, &item.Scope, &kind, &item.SnapshotID, &item.SourceJobID, &evidence, &item.CreatedAt); err != nil {
return nil, err
}
item.Evidence = decodeEvidenceObject(evidence)
item.Kind = governance.LibraryVersionKind(kind)
items = append(items, item)
}
return items, rows.Err()
}
// decodeEvidenceObject 把证据列解成对象:历史行里也有把证据存成"数组"的
// (早期任务把一组事件直接塞进 jsonb)。读原产地只关心字段,所以数组取最后一个对象,
// 解不动就返回 nil(调用方按"没有证据"处理,不会因此放宽判定)。
func decodeEvidenceObject(raw []byte) map[string]any {
trimmed := bytes.TrimSpace(raw)
if len(trimmed) == 0 {
return nil
}
if trimmed[0] == '{' {
var object map[string]any
if err := json.Unmarshal(trimmed, &object); err == nil {
return object
}
return nil
}
if trimmed[0] == '[' {
var items []map[string]any
if err := json.Unmarshal(trimmed, &items); err == nil && len(items) > 0 {
return items[len(items)-1]
}
}
return nil
}
// LibrarySnapshotKinds 返回这份快照在该 scope 下登记过的版本类型。
//
// 用途是判定"这份快照的清单身份应该等于谁":封存版本(test_sealed)的清单写的是
// **源**的 tenant/KB;预备份/生效/恢复版本写的是**目标**的 tenant/KB。没有登记过
// 就说明来源不明,调用方必须拒绝搬运(不能信快照自报)。
func (s *Store) LibrarySnapshotKinds(ctx context.Context, scope, snapshotID string) ([]string, error) {
if scope == "" || snapshotID == "" {
return nil, errors.New("snapshot kinds requires scope and snapshot")
}
rows, err := s.db.Query(ctx, `SELECT DISTINCT kind FROM library_versions WHERE scope=$1 AND snapshot_id=$2 AND (kind='test_sealed' OR target_id=$3)`, scope, snapshotID, s.libraryTargetID)
if err != nil {
return nil, fmt.Errorf("read snapshot provenance: %w", err)
}
defer rows.Close()
kinds := make([]string, 0, 4)
for rows.Next() {
var kind string
if err := rows.Scan(&kind); err != nil {
return nil, err
}
kinds = append(kinds, kind)
}
return kinds, rows.Err()
}
// SnapshotCandidate 是一份"本机可能需要有"的快照,以及它最近一次被引用的时间。
// CreatedAt 同时充当分页游标:按 (created_at, snapshot_id) 倒序,跨页不重不漏。
type SnapshotCandidate struct {
SnapshotID string
CreatedAt time.Time
}
// LibrarySnapshotCandidates 枚举该 scope 下**被引用过**的快照(分页,游标不是 OFFSET)。
//
// 为什么不能"只取最近 N 条成功作业的 snapshot_id":那会把回传永久卡在最早的 50 条上,
// 第 51 条以后的预备份、以及失败/待恢复作业留下的**恢复所需**预备份再也读不回来。
// 所以这里合并三处权威引用:
// - library_versions.snapshot_id(版本台账,含封存/预备份/生效/恢复);
// - library_jobs.snapshot_id(作业自己留下的预备份,含失败与 needs_restore);
// - library_jobs.target_snapshot_id(发布/恢复的输入快照,历史恢复要靠它寻址)。
//
// 游标语义:afterCreated/afterID 为空表示第一页;返回值不足 limit 即到底。
func (s *Store) LibrarySnapshotCandidates(ctx context.Context, scope string, afterCreated time.Time,
afterID string, limit int) ([]SnapshotCandidate, error) {
if scope == "" || limit <= 0 || limit > 500 {
return nil, errors.New("snapshot candidates requires scope and a bounded limit")
}
var cursor any
if !afterCreated.IsZero() {
cursor = afterCreated.UTC()
}
rows, err := s.db.Query(ctx, `
WITH referenced AS (
SELECT snapshot_id, created_at FROM library_versions WHERE scope=$1 AND (kind='test_sealed' OR target_id=$5) AND snapshot_id <> ''
UNION ALL
SELECT snapshot_id, created_at FROM library_jobs WHERE scope=$1 AND (kind='seal' OR target_id=$5) AND snapshot_id <> ''
UNION ALL
SELECT target_snapshot_id, created_at FROM library_jobs WHERE scope=$1 AND (kind='seal' OR target_id=$5) AND target_snapshot_id <> ''
), grouped AS (
SELECT snapshot_id, MAX(created_at) AS created_at FROM referenced GROUP BY snapshot_id
)
SELECT snapshot_id, created_at FROM grouped
WHERE $2::timestamptz IS NULL
OR created_at < $2::timestamptz
OR (created_at = $2::timestamptz AND snapshot_id < $3)
ORDER BY created_at DESC, snapshot_id DESC
LIMIT $4`, scope, cursor, afterID, limit, s.libraryTargetID)
if err != nil {
return nil, fmt.Errorf("list snapshot candidates: %w", err)
}
defer rows.Close()
items := make([]SnapshotCandidate, 0, limit)
for rows.Next() {
var item SnapshotCandidate
if err := rows.Scan(&item.SnapshotID, &item.CreatedAt); err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
// handoff 阶段的三个阶段(running/succeeded/failed)与业务作业行的所有权判定
// **必须在同一个事务里**:分开做时,读到的 owner 与自己写时已经不是同一个执行者,
// 迟到的旧 handoff 就能覆盖已经交给 release 的状态与阶段。
//
// 所有权只由 library_jobs.queue_job_id 决定(阶段行是否存在与所有权无关),
// 并且每次都在 `SELECT ... FOR UPDATE` 之后判定,跨表并发交接因此不能穿插。
func (s *Store) lockLibraryJob(ctx context.Context, tx pgx.Tx, libraryJobID, ownerQueueID string) (string, bool, error) {
var state string
var attached *string
if err := tx.QueryRow(ctx, `SELECT state, queue_job_id FROM library_jobs WHERE id=$1 FOR UPDATE`,
libraryJobID).Scan(&state, &attached); err != nil {
return "", false, fmt.Errorf("lock library job: %w", err)
}
current := ""
if attached != nil {
current = *attached
}
if current != ownerQueueID || (state != "pending" && state != "running") {
return state, false, nil
}
return state, true, nil
}
func upsertLibraryStage(ctx context.Context, tx pgx.Tx, libraryJobID, stage, state, detail string, now time.Time) error {
_, err := tx.Exec(ctx, `
INSERT INTO library_job_stages (job_id, stage, state, detail, updated_at)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (job_id, stage) DO UPDATE
SET state=EXCLUDED.state, detail=EXCLUDED.detail, updated_at=EXCLUDED.updated_at`,
libraryJobID, stage, state, detail, now.UTC())
if err != nil {
return fmt.Errorf("record library stage %s: %w", stage, err)
}
return nil
}
// BeginLibraryHandoff 在事务里创建/推进 handoff 阶段为 running,并确认所有权仍在。
// 返回 false 表示"这份作业已经不归本次 handoff 了",调用方不得再碰任何状态。
func (s *Store) BeginLibraryHandoff(ctx context.Context, libraryJobID, ownerQueueID, detail string,
now time.Time) (bool, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, owned, err := s.lockLibraryJob(ctx, tx, libraryJobID, ownerQueueID); err != nil {
return false, err
} else if !owned {
return false, nil
}
if err := upsertLibraryStage(ctx, tx, libraryJobID, "handoff", "running", detail, now); err != nil {
return false, err
}
if err := tx.Commit(ctx); err != nil {
return false, err
}
return true, nil
}
// CompleteLibraryHandoff 在**一个事务**里:确认所有权 → 投递发布队列作业 → 把业务作业
// 交接给该队列作业 → 把 handoff 阶段写成 succeeded。任何一步失败都整体回滚,
// 不会出现"作业已交给 release 但阶段还停在 running"或"阶段成功但作业没交接"。
func (s *Store) CompleteLibraryHandoff(ctx context.Context, libraryJobID, ownerQueueID string,
request jobs.EnqueueRequest, detail string, now time.Time) (bool, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, owned, err := s.lockLibraryJob(ctx, tx, libraryJobID, ownerQueueID); err != nil {
return false, err
} else if !owned {
return false, nil
}
if _, err := scanJob(tx.QueryRow(ctx, enqueueJobSQL, request.ID, request.Type, request.Payload,
request.Priority, request.AvailableAt, request.MaxAttempts, request.IdempotencyKey, now)); err != nil {
return false, fmt.Errorf("enqueue release job: %w", err)
}
tag, err := tx.Exec(ctx, `UPDATE library_jobs SET queue_job_id=$2, updated_at=$3
WHERE id=$1 AND queue_job_id=$4`, libraryJobID, request.ID, now.UTC(), ownerQueueID)
if err != nil {
return false, fmt.Errorf("hand over library job: %w", err)
}
if tag.RowsAffected() != 1 {
return false, errors.New("hand over library job: 所有权在事务内已变化,回滚")
}
if err := upsertLibraryStage(ctx, tx, libraryJobID, "handoff", "succeeded", detail, now); err != nil {
return false, err
}
if err := tx.Commit(ctx); err != nil {
return false, err
}
return true, nil
}
// FailLibraryHandoff 同一个事务里写"阶段失败 + 作业失败",不留半条记录。
func (s *Store) FailLibraryHandoff(ctx context.Context, libraryJobID, ownerQueueID, message string,
now time.Time) (bool, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, owned, err := s.lockLibraryJob(ctx, tx, libraryJobID, ownerQueueID); err != nil {
return false, err
} else if !owned {
return false, nil
}
if err := upsertLibraryStage(ctx, tx, libraryJobID, "handoff", "failed", message, now); err != nil {
return false, err
}
tag, err := tx.Exec(ctx, `
UPDATE library_jobs SET state='failed', stage='handoff', error=$3, recovery=$4,
updated_at=$2, finished_at=$2
WHERE id=$1 AND queue_job_id=$5 AND state IN ('pending','running')`,
libraryJobID, now.UTC(), message, "未进入维护窗口、未写任何知识;修正后重试", ownerQueueID)
if err != nil {
return false, fmt.Errorf("fail library handoff: %w", err)
}
if tag.RowsAffected() != 1 {
return false, errors.New("fail library handoff: 所有权在事务内已变化,回滚")
}
if err := tx.Commit(ctx); err != nil {
return false, err
}
return true, nil
}