package postgres import ( "bytes" "context" "encoding/json" "errors" "fmt" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" ) // 整库作业与版本:队列(jobs)负责租约重试,本文件负责业务身份、阶段与结果。 // 互斥与幂等都由数据库唯一索引保证,不依赖前端或单 worker 部署。 func (s *Store) CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error) { request := job.RequestDigest if request == "" { return governance.LibraryJob{}, false, errors.New("library job requires a request digest") } rawRequest, err := json.Marshal(map[string]any{"digest": request, "kind": string(job.Kind), "scope": job.Scope}) if err != nil { return governance.LibraryJob{}, false, err } var id string err = s.db.QueryRow(ctx, ` INSERT INTO library_jobs (id, scope, kind, state, stage, idempotency_key, request_digest, request, queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, operation_id, created_at, updated_at, target_id) VALUES ($1,$2,$3,'pending','',$4,$5,$6,$7,$8,$9,$10,'','',$11,$12,$12,$13) ON CONFLICT (idempotency_key) DO NOTHING RETURNING id`, job.ID, job.Scope, string(job.Kind), job.IdempotencyKey, request, rawRequest, job.QueueJobID, job.SnapshotID, job.TargetSnapshotID, job.Actor, job.OperationID, now.UTC(), s.libraryTargetID).Scan(&id) if err != nil { var pgErr *pgconn.PgError if errors.As(err, &pgErr) && pgErr.Code == "23505" { // 同一目标的写作业(sync/restore)共用一个唯一索引,跨 kind 互斥。 if pgErr.ConstraintName == "library_jobs_write_target" || pgErr.ConstraintName == "library_jobs_active" { return governance.LibraryJob{}, false, governance.ErrLibraryJobActive } } if !errors.Is(err, pgx.ErrNoRows) { return governance.LibraryJob{}, false, fmt.Errorf("create library job: %w", err) } // 幂等键已存在:必须返回**已存**作业,并核对请求体是否一致。 existing, loadErr := s.libraryJobByKey(ctx, job.IdempotencyKey) if loadErr != nil { return governance.LibraryJob{}, false, loadErr } if existing.RequestDigest != request { return existing, false, governance.ErrLibraryJobIdempotencyConflict } return existing, true, nil } for _, stage := range stages { if _, err := s.db.Exec(ctx, ` INSERT INTO library_job_stages (job_id, stage, state, detail, updated_at) VALUES ($1,$2,'pending','',$3) ON CONFLICT (job_id, stage) DO NOTHING`, id, stage, now.UTC()); err != nil { return governance.LibraryJob{}, false, fmt.Errorf("create library job stage: %w", err) } } created, err := s.LibraryJob(ctx, id) return created, false, err } func (s *Store) libraryJobByKey(ctx context.Context, key string) (governance.LibraryJob, error) { var id string if err := s.db.QueryRow(ctx, `SELECT id FROM library_jobs WHERE idempotency_key=$1`, key).Scan(&id); err != nil { if errors.Is(err, pgx.ErrNoRows) { return governance.LibraryJob{}, errors.New("library job idempotency key not found") } return governance.LibraryJob{}, fmt.Errorf("load library job by key: %w", err) } return s.LibraryJob(ctx, id) } func (s *Store) LibraryJob(ctx context.Context, id string) (governance.LibraryJob, error) { var job governance.LibraryJob var kind, state string var finished *time.Time if err := s.db.QueryRow(ctx, ` SELECT id, scope, kind, state, stage, idempotency_key, request_digest, queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, operation_id, attempt_epoch, created_at, updated_at, finished_at FROM library_jobs WHERE id=$1 AND (kind='seal' OR target_id=$2)`, id, s.libraryTargetID).Scan(&job.ID, &job.Scope, &kind, &state, &job.Stage, &job.IdempotencyKey, &job.RequestDigest, &job.QueueJobID, &job.SnapshotID, &job.TargetSnapshotID, &job.Actor, &job.Error, &job.Recovery, &job.OperationID, &job.AttemptEpoch, &job.CreatedAt, &job.UpdatedAt, &finished); err != nil { if errors.Is(err, pgx.ErrNoRows) { return governance.LibraryJob{}, err } return governance.LibraryJob{}, fmt.Errorf("load library job: %w", err) } job.Kind, job.State, job.FinishedAt = governance.LibraryJobKind(kind), governance.LibraryJobState(state), finished stages, err := s.libraryStages(ctx, id) if err != nil { return governance.LibraryJob{}, err } job.Stages = stages return job, nil } // LibraryReuseSource 是一条"这个目标的持久来源"记录:成功同步过的快照, // 或这个目标自己的预备份。资源复用只认它作来源证据。 type LibraryReuseSource struct { JobID string SnapshotID string Kind string // sync | prebackup FinishedAt time.Time } // LibraryReuseSources 读这个目标的持久来源(含已被恢复顶替的生效版本)。 // 同步来源要求同时存在 production_effective 版本登记:只写成功但版本没登记的 // 作业不算可审计来源。 func (s *Store) LibraryReuseSources(ctx context.Context, scope string, limit int) ([]LibraryReuseSource, error) { if limit < 1 || limit > 200 { limit = 100 } rows, err := s.db.Query(ctx, ` SELECT j.id, j.target_snapshot_id, 'sync', coalesce(j.finished_at, j.updated_at) FROM library_jobs j WHERE j.scope=$1 AND j.target_id=$3 AND j.kind='sync' AND j.state='succeeded' AND coalesce(j.target_snapshot_id,'')<>'' AND EXISTS (SELECT 1 FROM library_versions v WHERE v.scope=j.scope AND v.target_id=j.target_id AND v.kind='production_effective' AND v.snapshot_id=j.target_snapshot_id AND v.source_job_id=j.id) UNION ALL SELECT coalesce(v.source_job_id,''), v.snapshot_id, 'prebackup', v.created_at FROM library_versions v WHERE v.scope=$1 AND v.target_id=$3 AND v.kind='production_prebackup' AND coalesce(v.snapshot_id,'')<>'' ORDER BY 4 DESC LIMIT $2`, scope, limit, s.libraryTargetID) if err != nil { return nil, fmt.Errorf("load library reuse sources: %w", err) } defer rows.Close() sources := make([]LibraryReuseSource, 0, 8) for rows.Next() { var source LibraryReuseSource if err := rows.Scan(&source.JobID, &source.SnapshotID, &source.Kind, &source.FinishedAt); err != nil { return nil, fmt.Errorf("scan library reuse source: %w", err) } sources = append(sources, source) } return sources, rows.Err() } func (s *Store) libraryStages(ctx context.Context, id string) ([]governance.LibraryStage, error) { rows, err := s.db.Query(ctx, ` SELECT stage, state, total, completed, bytes_total, bytes_done, detail, updated_at FROM library_job_stages WHERE job_id=$1 ORDER BY updated_at, stage`, id) if err != nil { return nil, fmt.Errorf("load library job stages: %w", err) } defer rows.Close() result := make([]governance.LibraryStage, 0, 7) for rows.Next() { var stage governance.LibraryStage if err := rows.Scan(&stage.Stage, &stage.State, &stage.Total, &stage.Completed, &stage.BytesTotal, &stage.BytesDone, &stage.Detail, &stage.UpdatedAt); err != nil { return nil, fmt.Errorf("scan library job stage: %w", err) } result = append(result, stage) } return result, rows.Err() } // ListLibraryJobs 返回某目标最近的作业(按创建时间倒序),供页面在重启后恢复展示。 func (s *Store) ListLibraryJobs(ctx context.Context, scope string, limit int) ([]governance.LibraryJob, error) { if limit < 1 || limit > 50 { return nil, errors.New("library job list limit out of range") } rows, err := s.db.Query(ctx, `SELECT id FROM library_jobs WHERE scope=$1 AND (kind='seal' OR target_id=$3) ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit, s.libraryTargetID) if err != nil { return nil, fmt.Errorf("list library jobs: %w", err) } defer rows.Close() ids := make([]string, 0, limit) for rows.Next() { var id string if err := rows.Scan(&id); err != nil { return nil, fmt.Errorf("scan library job id: %w", err) } ids = append(ids, id) } if err := rows.Err(); err != nil { return nil, err } result := make([]governance.LibraryJob, 0, len(ids)) for _, id := range ids { job, err := s.LibraryJob(ctx, id) if err != nil { return nil, err } result = append(result, job) } return result, nil } // UpdateLibraryStage 只改调用方给了值的维度:nil 表示"这一维还没有可靠数据", // 保持 NULL,绝不写成 0。 func (s *Store) UpdateLibraryStage(ctx context.Context, jobID, stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error { tag, err := s.db.Exec(ctx, ` UPDATE library_job_stages SET state = CASE WHEN $3='' THEN state ELSE $3 END, total = COALESCE($4, total), completed = COALESCE($5, completed), bytes_total = COALESCE($6, bytes_total), bytes_done = COALESCE($7, bytes_done), detail = CASE WHEN $8='' THEN detail ELSE $8 END, updated_at = $9 WHERE job_id=$1 AND stage=$2`, jobID, stage, state, total, completed, bytesTotal, bytesDone, detail, now.UTC()) if err != nil { return fmt.Errorf("update library stage: %w", err) } if tag.RowsAffected() != 1 { return errors.New("library stage not found") } return nil } // AttachLibraryQueueJob 把队列作业号写到已建的业务作业上(两步创建,避免先入队后才发现互斥冲突)。 func (s *Store) AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error { tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET queue_job_id=$2, updated_at=$3 WHERE id=$1`, jobID, queueJobID, now.UTC()) if err != nil { return fmt.Errorf("attach library queue job: %w", err) } if tag.RowsAffected() != 1 { return errors.New("library job not found") } return nil } func (s *Store) CancelLibraryJob(ctx context.Context, jobID, reason string, now time.Time) error { return s.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobCancelled, "", reason, "", "", "", now) } func (s *Store) AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error { finished := any(nil) if state == governance.LibraryJobSucceeded || state == governance.LibraryJobFailed || state == governance.LibraryJobCancelled { finished = now.UTC() } tag, err := s.db.Exec(ctx, ` UPDATE library_jobs SET state=$2, stage = CASE WHEN $3='' THEN stage ELSE $3 END, error = CASE WHEN $4='' THEN error ELSE $4 END, recovery = CASE WHEN $5='' THEN recovery ELSE $5 END, snapshot_id = CASE WHEN $6='' THEN snapshot_id ELSE $6 END, target_snapshot_id = CASE WHEN $7='' THEN target_snapshot_id ELSE $7 END, updated_at=$8, finished_at=COALESCE($9, finished_at) WHERE id=$1`, jobID, string(state), stage, message, recovery, snapshotID, targetSnapshotID, now.UTC(), finished) if err != nil { return fmt.Errorf("advance library job: %w", err) } if tag.RowsAffected() != 1 { return errors.New("library job not found") } return nil } // RecordLibraryVersion 写入一条**不可变版本事件**,并在需要时推进 current 指针。 // // 与旧实现的区别(F02/F09 的根因): // - 事件 ID 含 operation:不同 operation 引用同一快照是两条事件,`source_job_id`/ // `evidence`/`created_at` 谁也不覆盖谁(旧实现是 `ON CONFLICT DO UPDATE` 整份替换)。 // - 同一 operation 重放是幂等:`ON CONFLICT (id) DO NOTHING`,已存行原样返回。 // - "谁当前生效"由 `library_current_versions` 表达,用单调次序做条件更新; // 更早登记的事实不能把指针拉回历史,旧 epoch 在写边界就被守卫拒绝(见 guard)。 // - 生效/恢复事件才动指针;预备份/封存只追加事件。 func (s *Store) RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error { targetID := s.libraryTargetID if version.Kind == governance.LibraryVersionTestSealed { targetID = "" } evidence, err := json.Marshal(version.Evidence) if err != nil { return fmt.Errorf("encode library version evidence: %w", err) } tx, err := s.db.Begin(ctx) if err != nil { return fmt.Errorf("begin library version: %w", err) } defer func() { _ = tx.Rollback(ctx) }() // 执行代次守卫:写事件的人必须还是当前代次(失租旧执行器不能登记"我成功了")。 if version.SourceJobID != "" && version.AttemptEpoch > 0 { if err := guardLibraryJobEpoch(ctx, tx, version.SourceJobID, version.AttemptEpoch, version.AttemptOwner); err != nil { return err } } // 事件身份 = (kind, scope, 快照, operation)。同一身份的重复登记只有两种合法形态: // · 同一执行者补登记**新增**的字段(预备份先登记、写集合随后补上); // · 完全没有新字段的纯重放。 // 所以冲突时只做**加法**:已有键永远赢(右操作数是已有 evidence),身份字段 // (source_job_id/operation_id/created_at)一格都不许改;换了执行者或换了意图的 // 登记在这里匹配不到行,什么也不写(F02/F09 不能靠 upsert 改写归属)。 // xmax = 0 才是真的新插入:重放同一份证据(或换了执行者被 WHERE 挡住)都不算新事实, // 否则一次纯重放就能把 current 再"推进"一次,把后来的操作顶掉。 var inserted bool err = tx.QueryRow(ctx, ` INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, target_id) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) ON CONFLICT (id) DO UPDATE SET evidence = EXCLUDED.evidence || library_versions.evidence WHERE library_versions.source_job_id = EXCLUDED.source_job_id AND library_versions.operation_id = EXCLUDED.operation_id AND library_versions.target_id = EXCLUDED.target_id RETURNING (xmax = 0)`, version.ID, version.Scope, string(version.Kind), version.SnapshotID, version.VersionLabel, version.SourceJobID, version.OperationID, version.AttemptEpoch, evidence, version.CreatedBy, version.CreatedAt.UTC(), targetID).Scan(&inserted) if errors.Is(err, pgx.ErrNoRows) { var existingTarget string if loadErr := tx.QueryRow(ctx, `SELECT target_id FROM library_versions WHERE id=$1`, version.ID).Scan(&existingTarget); loadErr != nil { return loadErr } if existingTarget != targetID { return errors.New("library version identity belongs to another target") } // 同一身份已经存在、但归属是别的执行者/别的意图:不覆盖、不报错,也不推进 current。 inserted = false err = nil } if err != nil { return fmt.Errorf("record library version event: %w", err) } if version.Kind == governance.LibraryVersionProductionEffective || version.Kind == governance.LibraryVersionRecoveryVerified { if err := advanceLibraryCurrent(ctx, tx, version, inserted, targetID); err != nil { return err } } return tx.Commit(ctx) } // advanceLibraryCurrent 用单调次序条件推进 current 指针。 // // 次序键 = (意图创建时间 | operation | epoch),**不是**事件登记时间(父的复核项): // 拿登记时间排序,一次迟到的旧 operation 重放(例如重跑几天前的同步)就会因为"我刚登记" // 而把 current 推回历史。意图的创建时间不会因为重放而变新,所以旧意图永远推不动 // 比它更新的意图。 // // 恢复要不要能把 current 指回更早的**内容**?要,但它是通过**新的意图**(kind=recover, // 创建于恢复时刻)做到的:内容更旧、意图更新,两者不矛盾。旧代次能不能写由 epoch // 守卫回答(见 libraryFence),不由时间回答。 func advanceLibraryCurrent(ctx context.Context, tx pgx.Tx, version governance.LibraryVersion, inserted bool, targetID string) error { orderAt := version.CreatedAt if version.OperationID != "" { var created *time.Time err := tx.QueryRow(ctx, `SELECT created_at FROM library_operations WHERE id=$1`, version.OperationID).Scan(&created) if errors.Is(err, pgx.ErrNoRows) { return fmt.Errorf("事件引用的 operation %s 不存在:拒绝推进 current", version.OperationID) } if err != nil { return fmt.Errorf("load library operation for current ordering: %w", err) } if created != nil { orderAt = *created } } orderKey := governance.CurrentOrderKey(orderAt, version.OperationID, version.AttemptEpoch) if !inserted { // 重放:事件已经在台账里。确认指针没有被别的更晚事实覆盖,否则只确认自己的历史。 var currentID string err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1 AND target_id=$2 FOR UPDATE`, version.Scope, targetID).Scan(¤tID) if errors.Is(err, pgx.ErrNoRows) { return nil } if err != nil { return fmt.Errorf("load library current pointer: %w", err) } return nil } tag, err := tx.Exec(ctx, ` INSERT INTO library_current_versions (scope, version_id, snapshot_id, operation_id, attempt_epoch, order_key, updated_at, target_id) VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (scope, target_id) DO UPDATE SET version_id=EXCLUDED.version_id, snapshot_id=EXCLUDED.snapshot_id, operation_id=EXCLUDED.operation_id, attempt_epoch=EXCLUDED.attempt_epoch, order_key=EXCLUDED.order_key, updated_at=EXCLUDED.updated_at WHERE EXCLUDED.order_key > library_current_versions.order_key`, version.Scope, version.ID, version.SnapshotID, version.OperationID, version.AttemptEpoch, orderKey, version.CreatedAt.UTC(), targetID) if err != nil { return fmt.Errorf("advance library current pointer: %w", err) } var currentID string if err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1 AND target_id=$2`, version.Scope, targetID).Scan(¤tID); err != nil { return fmt.Errorf("read back library current pointer: %w", err) } if tag.RowsAffected() == 0 && currentID != version.ID { return fmt.Errorf("%w(scope=%s:当前 %s,本条 %s)", governance.ErrLibraryCurrentStale, version.Scope, currentID, version.ID) } // superseded_at 只是展示元数据("这条已被顶替"),不改任何证据字段。 // 它跟着指针走:指针指向谁,谁就是未顶替的那一行。 if _, err := tx.Exec(ctx, ` UPDATE library_versions SET superseded_at = CASE WHEN id=$2 THEN NULL ELSE COALESCE(superseded_at, $3) END WHERE scope=$1 AND target_id=$4 AND kind IN ('production_effective','recovery_verified')`, version.Scope, currentID, version.CreatedAt.UTC(), targetID); err != nil { return fmt.Errorf("refresh library version supersede metadata: %w", err) } return nil } // InterruptedLibraryJobs 找出"业务作业 running、但队列里已没有活跃作业"的作业: // worker 被杀/重启后它们需要被接管,而不是被当成失败丢掉已完成的预备份。 func (s *Store) InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error) { // lease 过期才算中断:worker 刚被杀但 lease 未到期时,队列里还是 running, // 直接判"没中断"会让作业永远悬着;所以这里按 lease_until 判断。 rows, err := s.db.Query(ctx, ` SELECT lj.id FROM library_jobs lj LEFT JOIN jobs j ON j.id=lj.queue_job_id WHERE lj.kind IN ('sync','restore','baseline') AND ( -- 业务作业在跑、队列已经没了/停了/租约过期 (lj.state='running' AND (j.id IS NULL OR j.state IN ('failed','completed','cancelled','dead') OR (j.state IN ('pending','running','retry') AND j.lease_until IS NOT NULL AND j.lease_until < now()))) -- 业务作业已被前一轮接管置为 pending,但队列作业的租约也没了(worker 又死了) OR (lj.state='pending' AND j.state='running' AND j.lease_until IS NOT NULL AND j.lease_until < now()) ) AND lj.target_id=$1 ORDER BY lj.created_at`, s.libraryTargetID) if err != nil { return nil, fmt.Errorf("list interrupted library jobs: %w", err) } defer rows.Close() ids := make([]string, 0, 4) for rows.Next() { var id string if err := rows.Scan(&id); err != nil { return nil, err } ids = append(ids, id) } if err := rows.Err(); err != nil { return nil, err } jobs := make([]governance.LibraryJob, 0, len(ids)) for _, id := range ids { job, err := s.LibraryJob(ctx, id) if err != nil { return nil, err } jobs = append(jobs, job) } return jobs, nil } // PrepareLibraryJobResume 把**同一个业务作业**放回 pending,供续作/接管重新执行。 // // 为什么是原作业而不是新开一行(R3):维护窗口 owner_job_id、freeze journal 文件名、 // 预备份与写集合的身份全部按业务作业 id 落在持久层。新开 `-resume` 作业只能继承 // operation_id,其它所有者都还指向旧作业——AcquireInstanceMaintenance 会拒绝它, // journal/预备份也查不到,于是"续作"只是一个漂亮的新行。 // // 条件更新:只有 failed/cancelled 才允许置回 pending;终态时间与错误清零, // 旧 attempt 行保留(历史不可变)。代次由 BeginLibraryAttempt 在 worker 起手时递增。 func (s *Store) PrepareLibraryJobResume(ctx context.Context, jobID string, now time.Time) (governance.LibraryJob, bool, error) { if _, err := s.LibraryJob(ctx, jobID); err != nil { return governance.LibraryJob{}, false, err } tx, err := s.db.Begin(ctx) if err != nil { return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err) } defer func() { _ = tx.Rollback(ctx) }() // 续作是人工发起的接管:作业已经 failed/cancelled,它那一代执行者不可能还在合法地写。 // 必须**显式结束**那一代(行保留、标 abandoned),否则它的租约还没过期, // BeginLibraryAttempt 会认为"有活跃执行者"而拒绝新代次——续作就永远开不起来。 if _, err := tx.Exec(ctx, ` UPDATE library_attempts SET state='abandoned', finished_at=$2, outcome='superseded by operator resume', lease_until=NULL WHERE job_id=$1 AND state IN ('claimed','running')`, jobID, now.UTC()); err != nil { return governance.LibraryJob{}, false, fmt.Errorf("retire attempts for resume: %w", err) } tag, err := tx.Exec(ctx, ` UPDATE library_jobs SET state='pending', error='', recovery='', finished_at=NULL, updated_at=$2, lease_owner='', lease_until=NULL WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC()) if err != nil { return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err) } if err := tx.Commit(ctx); err != nil { return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err) } job, loadErr := s.LibraryJob(ctx, jobID) if loadErr != nil { return governance.LibraryJob{}, false, loadErr } return job, tag.RowsAffected() == 1, nil } // ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零, // 否则 attempt '' AND NOT EXISTS ( SELECT 1 FROM jobs AS j WHERE j.id = lj.queue_job_id AND j.state IN ('pending','running','retry') )`, now.UTC(), kinds, s.libraryTargetID); err != nil { return 0, fmt.Errorf("reconcile library jobs: %w", err) } // 建了业务作业但队列投递没完成(queue_job_id 为空)超过 5 分钟,也是孤儿。 tag, err := s.db.Exec(ctx, ` UPDATE library_jobs SET state='failed', error='队列投递未完成,作业从未开始', recovery='无需恢复来源;可直接重发', updated_at=$1, finished_at=$1 WHERE state IN ('pending','running') AND queue_job_id='' AND kind = ANY($3) AND (kind='seal' OR target_id=$4) AND created_at < $2`, now.UTC(), now.Add(-5*time.Minute).UTC(), kinds, s.libraryTargetID) if err != nil { return 0, fmt.Errorf("reconcile unattached library jobs: %w", err) } return int(tag.RowsAffected()), nil } // LibraryVersionByRef 按 (scope, kind, 作业, 快照) 精确定位一条版本记录。 // // 为什么不能"取最近 N 条再在内存里筛":那是窗口,历史一旦超过窗口上限, // 登记/接管/恢复就会把"存在"读成"不存在"——真实现场是写集合的引用无法登记, // 报"读不到现有版本证据 / library version limit out of range"。 // 身份查询必须走索引条件,不许依赖排序窗口(把上限从 200 改成 100 只是换个数)。 // // 同一 (scope, kind, snapshot) 只该有一行(id 由这三者决定,写入是 upsert)。 // 命中多行、或命中的那行归别的作业所有,都说明台账不唯一:返回错误让调用方 // 拒绝,不许挑一条"看起来对"的记录去恢复生产。 func (s *Store) LibraryVersionByRef(ctx context.Context, scope string, kind governance.LibraryVersionKind, sourceJobID, snapshotID string) (governance.LibraryVersion, bool, error) { if scope == "" || kind == "" || sourceJobID == "" || snapshotID == "" { return governance.LibraryVersion{}, false, errors.New("library version ref requires scope, kind, job and snapshot") } rows, err := s.db.Query(ctx, ` SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at FROM library_versions WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4 AND (kind='test_sealed' OR target_id=$5) ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), sourceJobID, snapshotID, s.libraryTargetID) if err != nil { return governance.LibraryVersion{}, false, fmt.Errorf("load library version by ref: %w", err) } items, err := scanLibraryVersions(rows, 2) if err != nil { return governance.LibraryVersion{}, false, err } if len(items) > 1 { return governance.LibraryVersion{}, false, fmt.Errorf("同一身份存在 %d 条版本记录(作业 %s,快照 %s):拒绝挑选", len(items), sourceJobID, snapshotID) } if len(items) == 1 { return items[0], true, nil } // 没命中时要区分"确实没有"与"这一行归别的作业":后者是身份冲突, // 当成"没有记录"就会允许调用方覆盖别人的台账。 var owner string err = s.db.QueryRow(ctx, ` SELECT source_job_id FROM library_versions WHERE scope=$1 AND kind=$2 AND snapshot_id=$3 AND (kind='test_sealed' OR target_id=$4) ORDER BY created_at DESC LIMIT 1`, scope, string(kind), snapshotID, s.libraryTargetID).Scan(&owner) if errors.Is(err, pgx.ErrNoRows) { return governance.LibraryVersion{}, false, nil } if err != nil { return governance.LibraryVersion{}, false, fmt.Errorf("load library version owner: %w", err) } return governance.LibraryVersion{}, false, fmt.Errorf("版本记录(快照 %s)属于作业 %s,不是 %s:拒绝借用", snapshotID, owner, sourceJobID) } // scanLibraryVersions 是版本行唯一的扫描实现:两个查询(窗口列表与按身份精确查) // 必须对同一种形状解码,否则"查得到"和"读得对"会分叉。 func scanLibraryVersions(rows pgx.Rows, capacity int) ([]governance.LibraryVersion, error) { defer rows.Close() result := make([]governance.LibraryVersion, 0, capacity) for rows.Next() { var item governance.LibraryVersion var kind string var evidence []byte if err := rows.Scan(&item.ID, &item.Scope, &kind, &item.SnapshotID, &item.VersionLabel, &item.SourceJobID, &item.OperationID, &item.AttemptEpoch, &evidence, &item.CreatedBy, &item.CreatedAt, &item.SupersededAt); err != nil { return nil, fmt.Errorf("scan library version: %w", err) } item.Kind = governance.LibraryVersionKind(kind) if len(evidence) > 0 { _ = json.Unmarshal(evidence, &item.Evidence) } result = append(result, item) } return result, rows.Err() } // LibraryVersions 是**分页展示**用的窗口列表(1..100),语义是"最近 N 条"。 // 任何"当前生效 / 本快照是否生效过"的判断都不许用它:窗口一满就会把存在读成不存在。 func (s *Store) LibraryVersions(ctx context.Context, scope string, limit int) ([]governance.LibraryVersion, error) { if limit < 1 || limit > 100 { return nil, errors.New("library version limit out of range") } rows, err := s.db.Query(ctx, ` SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at FROM library_versions WHERE scope=$1 AND (kind='test_sealed' OR target_id=$3) ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit, s.libraryTargetID) if err != nil { return nil, fmt.Errorf("list library versions: %w", err) } return scanLibraryVersions(rows, limit) } // LibraryCurrentVersion 返回这个 scope **当前生效**的事实(由指针表回答)。 // // 三个理由必须一起满足: // // · 不能用窗口列表——生效事实可能很久以前登记,之后写进来的预备份/封存会把它挤出 // 任何"最近 N 条",接管/基线/闸口就会把"已生效"读成"没有生效"。 // · 不能靠扫 library_versions.kind='production_effective' 挑未顶替行——事件不可变, // "谁是当前"是指针的事实,不是事件行的可变字段(F02)。 // · 指针每 (scope, target_id) 至多一行,历史归属不明的空身份不归新目标。 // // 指针可能指向 production_effective,也可能指向 recovery_verified(按预备份恢复后 // 的当前内容身份)——调用方只能用它回答"当前是什么内容",不能假设它是一次成功发布。 func (s *Store) LibraryCurrentVersion(ctx context.Context, scope string) (governance.LibraryVersion, bool, error) { if scope == "" { return governance.LibraryVersion{}, false, errors.New("library current version requires scope") } rows, err := s.db.Query(ctx, ` SELECT v.id, v.scope, v.kind, v.snapshot_id, v.version_label, v.source_job_id, v.operation_id, v.attempt_epoch, v.evidence, v.created_by, v.created_at, v.superseded_at FROM library_current_versions c JOIN library_versions v ON v.id = c.version_id WHERE c.scope=$1 AND c.target_id=$2 AND v.target_id=c.target_id`, scope, s.libraryTargetID) if err != nil { return governance.LibraryVersion{}, false, fmt.Errorf("load current library version: %w", err) } items, err := scanLibraryVersions(rows, 2) if err != nil { return governance.LibraryVersion{}, false, err } if len(items) > 1 { return governance.LibraryVersion{}, false, fmt.Errorf("scope=%s 的当前指针指向 %d 行事件:台账被写坏,拒绝挑选", scope, len(items)) } if len(items) == 1 { return items[0], true, nil } return governance.LibraryVersion{}, false, nil } // LibraryVersionOfSnapshot 按 (scope, kind, 快照) 取这个快照**自己的**登记记录(不限作业)。 // // 用途是"本快照曾经生效过吗/是否已被顶替":版本 id 由 (kind, scope, 快照) 决定, // 一个快照在一种类型下只该有一行,作业归属会随恢复/重放被改写,所以不能用作业当条件。 // 命中多行同样拒绝挑选。 func (s *Store) LibraryVersionOfSnapshot(ctx context.Context, scope string, kind governance.LibraryVersionKind, snapshotID string) (governance.LibraryVersion, bool, error) { if scope == "" || kind == "" || snapshotID == "" { return governance.LibraryVersion{}, false, errors.New("library version snapshot requires scope, kind and snapshot") } rows, err := s.db.Query(ctx, ` SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at FROM library_versions WHERE scope=$1 AND kind=$2 AND snapshot_id=$3 AND (kind='test_sealed' OR target_id=$4) ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), snapshotID, s.libraryTargetID) if err != nil { return governance.LibraryVersion{}, false, fmt.Errorf("load library version of snapshot: %w", err) } found, err := scanLibraryVersions(rows, 2) if err != nil { return governance.LibraryVersion{}, false, err } // 同一快照可以被多次意图引用,每次 operation 一条事件,这是**正常历史**; // 但"两条同身份的行"是台账被写坏(或迁移前没有 operation 身份的未知证据), // 这时挑选任何一条都是替调用方做决定,必须拒绝(未知证据 fail closed)。 if len(found) == 1 { return found[0], true, nil } if len(found) >= 2 { if found[0].OperationID == "" || found[1].OperationID == "" { return governance.LibraryVersion{}, false, fmt.Errorf( "快照 %s 在 %s 下有 %d 条无执行身份的记录:拒绝挑选", snapshotID, kind, len(found)) } if found[0].OperationID == found[1].OperationID { return governance.LibraryVersion{}, false, fmt.Errorf( "快照 %s 在 %s 下同一意图登记了多行:拒绝挑选", snapshotID, kind) } return found[0], true, nil } return governance.LibraryVersion{}, false, nil } // CreateLibraryAuthorization 记录一次生产发布授权;同一 (scope,snapshot,digest) 只保留最新有效。 func (s *Store) CreateLibraryAuthorization(ctx context.Context, item governance.LibraryReleaseAuthorization) error { if item.ID == "" || item.Scope == "" || item.SnapshotID == "" || item.ParamsDigest == "" || item.ExpiresAt.IsZero() { return errors.New("invalid library release authorization") } _, err := s.db.Exec(ctx, ` INSERT INTO library_release_authorizations (id, scope, snapshot_id, params_digest, authorized_by, expires_at, created_at, target_id) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`, item.ID, item.Scope, item.SnapshotID, item.ParamsDigest, item.AuthorizedBy, item.ExpiresAt.UTC(), item.CreatedAt.UTC(), s.libraryTargetID) if err != nil { return fmt.Errorf("create library release authorization: %w", err) } return nil } // ActiveLibraryAuthorization 只返回"未撤销、未过期、且参数摘要完全一致"的授权。 // 参数一变就查不到,自然拒绝,而不是靠调用方记得复核。 func (s *Store) ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, paramsDigest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error) { var item governance.LibraryReleaseAuthorization err := s.db.QueryRow(ctx, ` SELECT id, scope, snapshot_id, params_digest, authorized_by, expires_at, revoked_at, created_at FROM library_release_authorizations WHERE scope=$1 AND snapshot_id=$2 AND params_digest=$3 AND target_id=$5 AND revoked_at IS NULL AND expires_at > $4 ORDER BY expires_at DESC, created_at DESC LIMIT 1`, scope, snapshotID, paramsDigest, now.UTC(), s.libraryTargetID).Scan(&item.ID, &item.Scope, &item.SnapshotID, &item.ParamsDigest, &item.AuthorizedBy, &item.ExpiresAt, &item.RevokedAt, &item.CreatedAt) if errors.Is(err, pgx.ErrNoRows) { return governance.LibraryReleaseAuthorization{}, false, nil } if err != nil { return governance.LibraryReleaseAuthorization{}, false, fmt.Errorf("load library authorization: %w", err) } return item, true, nil } func (s *Store) RevokeLibraryAuthorization(ctx context.Context, id string, now time.Time) error { tag, err := s.db.Exec(ctx, `UPDATE library_release_authorizations SET revoked_at=$2 WHERE id=$1 AND target_id=$3 AND revoked_at IS NULL`, id, now.UTC(), s.libraryTargetID) if err != nil { return fmt.Errorf("revoke library authorization: %w", err) } if tag.RowsAffected() != 1 { return errors.New("library authorization not found or already revoked") } return nil } func (s *Store) LibraryAuthorizations(ctx context.Context, scope string, limit int) ([]governance.LibraryReleaseAuthorization, error) { if limit < 1 || limit > 100 { return nil, errors.New("library authorization limit out of range") } rows, err := s.db.Query(ctx, ` SELECT id, scope, snapshot_id, params_digest, authorized_by, expires_at, revoked_at, created_at FROM library_release_authorizations WHERE scope=$1 AND target_id=$3 ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit, s.libraryTargetID) if err != nil { return nil, fmt.Errorf("list library authorizations: %w", err) } defer rows.Close() result := make([]governance.LibraryReleaseAuthorization, 0, limit) for rows.Next() { var item governance.LibraryReleaseAuthorization if err := rows.Scan(&item.ID, &item.Scope, &item.SnapshotID, &item.ParamsDigest, &item.AuthorizedBy, &item.ExpiresAt, &item.RevokedAt, &item.CreatedAt); err != nil { return nil, fmt.Errorf("scan library authorization: %w", err) } result = append(result, item) } return result, rows.Err() } // LibraryVersionsBySnapshot 返回这份快照在该 scope 下的全部版本事件(含证据)。 // // 原产地不能只看 kind:目标权威基线也是 production_effective,但它是**目标侧的真实捕获** // (证据里 action=target_baseline_init / origin=target_baseline)。所以判定必须读到证据, // 不能把"是 effective"直接当成"来自发布"或"来自目标"。 func (s *Store) LibraryVersionsBySnapshot(ctx context.Context, scope, snapshotID string) ([]governance.LibraryVersion, error) { if scope == "" || snapshotID == "" { return nil, errors.New("snapshot versions requires scope and snapshot") } rows, err := s.db.Query(ctx, ` SELECT id, scope, kind, snapshot_id, source_job_id, COALESCE(evidence, '{}'::jsonb), created_at FROM library_versions WHERE scope=$1 AND snapshot_id=$2 AND (kind='test_sealed' OR target_id=$3) ORDER BY created_at`, scope, snapshotID, s.libraryTargetID) if err != nil { return nil, fmt.Errorf("read snapshot versions: %w", err) } defer rows.Close() items := make([]governance.LibraryVersion, 0, 4) for rows.Next() { var item governance.LibraryVersion var kind string var evidence []byte if err := rows.Scan(&item.ID, &item.Scope, &kind, &item.SnapshotID, &item.SourceJobID, &evidence, &item.CreatedAt); err != nil { return nil, err } item.Evidence = decodeEvidenceObject(evidence) item.Kind = governance.LibraryVersionKind(kind) items = append(items, item) } return items, rows.Err() } // decodeEvidenceObject 把证据列解成对象:历史行里也有把证据存成"数组"的 // (早期任务把一组事件直接塞进 jsonb)。读原产地只关心字段,所以数组取最后一个对象, // 解不动就返回 nil(调用方按"没有证据"处理,不会因此放宽判定)。 func decodeEvidenceObject(raw []byte) map[string]any { trimmed := bytes.TrimSpace(raw) if len(trimmed) == 0 { return nil } if trimmed[0] == '{' { var object map[string]any if err := json.Unmarshal(trimmed, &object); err == nil { return object } return nil } if trimmed[0] == '[' { var items []map[string]any if err := json.Unmarshal(trimmed, &items); err == nil && len(items) > 0 { return items[len(items)-1] } } return nil } // LibrarySnapshotKinds 返回这份快照在该 scope 下登记过的版本类型。 // // 用途是判定"这份快照的清单身份应该等于谁":封存版本(test_sealed)的清单写的是 // **源**的 tenant/KB;预备份/生效/恢复版本写的是**目标**的 tenant/KB。没有登记过 // 就说明来源不明,调用方必须拒绝搬运(不能信快照自报)。 func (s *Store) LibrarySnapshotKinds(ctx context.Context, scope, snapshotID string) ([]string, error) { if scope == "" || snapshotID == "" { return nil, errors.New("snapshot kinds requires scope and snapshot") } rows, err := s.db.Query(ctx, `SELECT DISTINCT kind FROM library_versions WHERE scope=$1 AND snapshot_id=$2 AND (kind='test_sealed' OR target_id=$3)`, scope, snapshotID, s.libraryTargetID) if err != nil { return nil, fmt.Errorf("read snapshot provenance: %w", err) } defer rows.Close() kinds := make([]string, 0, 4) for rows.Next() { var kind string if err := rows.Scan(&kind); err != nil { return nil, err } kinds = append(kinds, kind) } return kinds, rows.Err() } // SnapshotCandidate 是一份"本机可能需要有"的快照,以及它最近一次被引用的时间。 // CreatedAt 同时充当分页游标:按 (created_at, snapshot_id) 倒序,跨页不重不漏。 type SnapshotCandidate struct { SnapshotID string CreatedAt time.Time } // LibrarySnapshotCandidates 枚举该 scope 下**被引用过**的快照(分页,游标不是 OFFSET)。 // // 为什么不能"只取最近 N 条成功作业的 snapshot_id":那会把回传永久卡在最早的 50 条上, // 第 51 条以后的预备份、以及失败/待恢复作业留下的**恢复所需**预备份再也读不回来。 // 所以这里合并三处权威引用: // - library_versions.snapshot_id(版本台账,含封存/预备份/生效/恢复); // - library_jobs.snapshot_id(作业自己留下的预备份,含失败与 needs_restore); // - library_jobs.target_snapshot_id(发布/恢复的输入快照,历史恢复要靠它寻址)。 // // 游标语义:afterCreated/afterID 为空表示第一页;返回值不足 limit 即到底。 func (s *Store) LibrarySnapshotCandidates(ctx context.Context, scope string, afterCreated time.Time, afterID string, limit int) ([]SnapshotCandidate, error) { if scope == "" || limit <= 0 || limit > 500 { return nil, errors.New("snapshot candidates requires scope and a bounded limit") } var cursor any if !afterCreated.IsZero() { cursor = afterCreated.UTC() } rows, err := s.db.Query(ctx, ` WITH referenced AS ( SELECT snapshot_id, created_at FROM library_versions WHERE scope=$1 AND (kind='test_sealed' OR target_id=$5) AND snapshot_id <> '' UNION ALL SELECT snapshot_id, created_at FROM library_jobs WHERE scope=$1 AND (kind='seal' OR target_id=$5) AND snapshot_id <> '' UNION ALL SELECT target_snapshot_id, created_at FROM library_jobs WHERE scope=$1 AND (kind='seal' OR target_id=$5) AND target_snapshot_id <> '' ), grouped AS ( SELECT snapshot_id, MAX(created_at) AS created_at FROM referenced GROUP BY snapshot_id ) SELECT snapshot_id, created_at FROM grouped WHERE $2::timestamptz IS NULL OR created_at < $2::timestamptz OR (created_at = $2::timestamptz AND snapshot_id < $3) ORDER BY created_at DESC, snapshot_id DESC LIMIT $4`, scope, cursor, afterID, limit, s.libraryTargetID) if err != nil { return nil, fmt.Errorf("list snapshot candidates: %w", err) } defer rows.Close() items := make([]SnapshotCandidate, 0, limit) for rows.Next() { var item SnapshotCandidate if err := rows.Scan(&item.SnapshotID, &item.CreatedAt); err != nil { return nil, err } items = append(items, item) } return items, rows.Err() } // handoff 阶段的三个阶段(running/succeeded/failed)与业务作业行的所有权判定 // **必须在同一个事务里**:分开做时,读到的 owner 与自己写时已经不是同一个执行者, // 迟到的旧 handoff 就能覆盖已经交给 release 的状态与阶段。 // // 所有权只由 library_jobs.queue_job_id 决定(阶段行是否存在与所有权无关), // 并且每次都在 `SELECT ... FOR UPDATE` 之后判定,跨表并发交接因此不能穿插。 func (s *Store) lockLibraryJob(ctx context.Context, tx pgx.Tx, libraryJobID, ownerQueueID string) (string, bool, error) { var state string var attached *string if err := tx.QueryRow(ctx, `SELECT state, queue_job_id FROM library_jobs WHERE id=$1 FOR UPDATE`, libraryJobID).Scan(&state, &attached); err != nil { return "", false, fmt.Errorf("lock library job: %w", err) } current := "" if attached != nil { current = *attached } if current != ownerQueueID || (state != "pending" && state != "running") { return state, false, nil } return state, true, nil } func upsertLibraryStage(ctx context.Context, tx pgx.Tx, libraryJobID, stage, state, detail string, now time.Time) error { _, err := tx.Exec(ctx, ` INSERT INTO library_job_stages (job_id, stage, state, detail, updated_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (job_id, stage) DO UPDATE SET state=EXCLUDED.state, detail=EXCLUDED.detail, updated_at=EXCLUDED.updated_at`, libraryJobID, stage, state, detail, now.UTC()) if err != nil { return fmt.Errorf("record library stage %s: %w", stage, err) } return nil } // BeginLibraryHandoff 在事务里创建/推进 handoff 阶段为 running,并确认所有权仍在。 // 返回 false 表示"这份作业已经不归本次 handoff 了",调用方不得再碰任何状态。 func (s *Store) BeginLibraryHandoff(ctx context.Context, libraryJobID, ownerQueueID, detail string, now time.Time) (bool, error) { tx, err := s.db.Begin(ctx) if err != nil { return false, err } defer func() { _ = tx.Rollback(ctx) }() if _, owned, err := s.lockLibraryJob(ctx, tx, libraryJobID, ownerQueueID); err != nil { return false, err } else if !owned { return false, nil } if err := upsertLibraryStage(ctx, tx, libraryJobID, "handoff", "running", detail, now); err != nil { return false, err } if err := tx.Commit(ctx); err != nil { return false, err } return true, nil } // CompleteLibraryHandoff 在**一个事务**里:确认所有权 → 投递发布队列作业 → 把业务作业 // 交接给该队列作业 → 把 handoff 阶段写成 succeeded。任何一步失败都整体回滚, // 不会出现"作业已交给 release 但阶段还停在 running"或"阶段成功但作业没交接"。 func (s *Store) CompleteLibraryHandoff(ctx context.Context, libraryJobID, ownerQueueID string, request jobs.EnqueueRequest, detail string, now time.Time) (bool, error) { tx, err := s.db.Begin(ctx) if err != nil { return false, err } defer func() { _ = tx.Rollback(ctx) }() if _, owned, err := s.lockLibraryJob(ctx, tx, libraryJobID, ownerQueueID); err != nil { return false, err } else if !owned { return false, nil } if _, err := scanJob(tx.QueryRow(ctx, enqueueJobSQL, request.ID, request.Type, request.Payload, request.Priority, request.AvailableAt, request.MaxAttempts, request.IdempotencyKey, now)); err != nil { return false, fmt.Errorf("enqueue release job: %w", err) } tag, err := tx.Exec(ctx, `UPDATE library_jobs SET queue_job_id=$2, updated_at=$3 WHERE id=$1 AND queue_job_id=$4`, libraryJobID, request.ID, now.UTC(), ownerQueueID) if err != nil { return false, fmt.Errorf("hand over library job: %w", err) } if tag.RowsAffected() != 1 { return false, errors.New("hand over library job: 所有权在事务内已变化,回滚") } if err := upsertLibraryStage(ctx, tx, libraryJobID, "handoff", "succeeded", detail, now); err != nil { return false, err } if err := tx.Commit(ctx); err != nil { return false, err } return true, nil } // FailLibraryHandoff 同一个事务里写"阶段失败 + 作业失败",不留半条记录。 func (s *Store) FailLibraryHandoff(ctx context.Context, libraryJobID, ownerQueueID, message string, now time.Time) (bool, error) { tx, err := s.db.Begin(ctx) if err != nil { return false, err } defer func() { _ = tx.Rollback(ctx) }() if _, owned, err := s.lockLibraryJob(ctx, tx, libraryJobID, ownerQueueID); err != nil { return false, err } else if !owned { return false, nil } if err := upsertLibraryStage(ctx, tx, libraryJobID, "handoff", "failed", message, now); err != nil { return false, err } tag, err := tx.Exec(ctx, ` UPDATE library_jobs SET state='failed', stage='handoff', error=$3, recovery=$4, updated_at=$2, finished_at=$2 WHERE id=$1 AND queue_job_id=$5 AND state IN ('pending','running')`, libraryJobID, now.UTC(), message, "未进入维护窗口、未写任何知识;修正后重试", ownerQueueID) if err != nil { return false, fmt.Errorf("fail library handoff: %w", err) } if tag.RowsAffected() != 1 { return false, errors.New("fail library handoff: 所有权在事务内已变化,回滚") } if err := tx.Commit(ctx); err != nil { return false, err } return true, nil }