1080 lines
50 KiB
Go
1080 lines
50 KiB
Go
package postgres
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
"github.com/jackc/pgx/v5/pgconn"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
)
|
||
|
||
// 整库作业与版本:队列(jobs)负责租约重试,本文件负责业务身份、阶段与结果。
|
||
// 互斥与幂等都由数据库唯一索引保证,不依赖前端或单 worker 部署。
|
||
|
||
func (s *Store) CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error) {
|
||
request := job.RequestDigest
|
||
if request == "" {
|
||
return governance.LibraryJob{}, false, errors.New("library job requires a request digest")
|
||
}
|
||
rawRequest, err := json.Marshal(map[string]any{"digest": request, "kind": string(job.Kind), "scope": job.Scope})
|
||
if err != nil {
|
||
return governance.LibraryJob{}, false, err
|
||
}
|
||
var id string
|
||
err = s.db.QueryRow(ctx, `
|
||
INSERT INTO library_jobs (id, scope, kind, state, stage, idempotency_key, request_digest, request,
|
||
queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, operation_id, created_at, updated_at, target_id)
|
||
VALUES ($1,$2,$3,'pending','',$4,$5,$6,$7,$8,$9,$10,'','',$11,$12,$12,$13)
|
||
ON CONFLICT (idempotency_key) DO NOTHING
|
||
RETURNING id`,
|
||
job.ID, job.Scope, string(job.Kind), job.IdempotencyKey, request, rawRequest,
|
||
job.QueueJobID, job.SnapshotID, job.TargetSnapshotID, job.Actor, job.OperationID, now.UTC(), s.libraryTargetID).Scan(&id)
|
||
if err != nil {
|
||
var pgErr *pgconn.PgError
|
||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||
// 同一目标的写作业(sync/restore)共用一个唯一索引,跨 kind 互斥。
|
||
if pgErr.ConstraintName == "library_jobs_write_target" || pgErr.ConstraintName == "library_jobs_active" {
|
||
return governance.LibraryJob{}, false, governance.ErrLibraryJobActive
|
||
}
|
||
}
|
||
if !errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryJob{}, false, fmt.Errorf("create library job: %w", err)
|
||
}
|
||
// 幂等键已存在:必须返回**已存**作业,并核对请求体是否一致。
|
||
existing, loadErr := s.libraryJobByKey(ctx, job.IdempotencyKey)
|
||
if loadErr != nil {
|
||
return governance.LibraryJob{}, false, loadErr
|
||
}
|
||
if existing.RequestDigest != request {
|
||
return existing, false, governance.ErrLibraryJobIdempotencyConflict
|
||
}
|
||
return existing, true, nil
|
||
}
|
||
for _, stage := range stages {
|
||
if _, err := s.db.Exec(ctx, `
|
||
INSERT INTO library_job_stages (job_id, stage, state, detail, updated_at)
|
||
VALUES ($1,$2,'pending','',$3)
|
||
ON CONFLICT (job_id, stage) DO NOTHING`, id, stage, now.UTC()); err != nil {
|
||
return governance.LibraryJob{}, false, fmt.Errorf("create library job stage: %w", err)
|
||
}
|
||
}
|
||
created, err := s.LibraryJob(ctx, id)
|
||
return created, false, err
|
||
}
|
||
|
||
func (s *Store) libraryJobByKey(ctx context.Context, key string) (governance.LibraryJob, error) {
|
||
var id string
|
||
if err := s.db.QueryRow(ctx, `SELECT id FROM library_jobs WHERE idempotency_key=$1`, key).Scan(&id); err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryJob{}, errors.New("library job idempotency key not found")
|
||
}
|
||
return governance.LibraryJob{}, fmt.Errorf("load library job by key: %w", err)
|
||
}
|
||
return s.LibraryJob(ctx, id)
|
||
}
|
||
|
||
func (s *Store) LibraryJob(ctx context.Context, id string) (governance.LibraryJob, error) {
|
||
var job governance.LibraryJob
|
||
var kind, state string
|
||
var finished *time.Time
|
||
if err := s.db.QueryRow(ctx, `
|
||
SELECT id, scope, kind, state, stage, idempotency_key, request_digest, queue_job_id, snapshot_id,
|
||
target_snapshot_id, actor, error, recovery, operation_id, attempt_epoch, created_at, updated_at, finished_at
|
||
FROM library_jobs WHERE id=$1 AND (kind='seal' OR target_id=$2)`, id, s.libraryTargetID).Scan(&job.ID, &job.Scope, &kind, &state, &job.Stage, &job.IdempotencyKey,
|
||
&job.RequestDigest, &job.QueueJobID, &job.SnapshotID, &job.TargetSnapshotID, &job.Actor, &job.Error,
|
||
&job.Recovery, &job.OperationID, &job.AttemptEpoch, &job.CreatedAt, &job.UpdatedAt, &finished); err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryJob{}, err
|
||
}
|
||
return governance.LibraryJob{}, fmt.Errorf("load library job: %w", err)
|
||
}
|
||
job.Kind, job.State, job.FinishedAt = governance.LibraryJobKind(kind), governance.LibraryJobState(state), finished
|
||
stages, err := s.libraryStages(ctx, id)
|
||
if err != nil {
|
||
return governance.LibraryJob{}, err
|
||
}
|
||
job.Stages = stages
|
||
return job, nil
|
||
}
|
||
|
||
// LibraryReuseSource 是一条"这个目标的持久来源"记录:成功同步过的快照,
|
||
// 或这个目标自己的预备份。资源复用只认它作来源证据。
|
||
type LibraryReuseSource struct {
|
||
JobID string
|
||
SnapshotID string
|
||
Kind string // sync | prebackup
|
||
FinishedAt time.Time
|
||
}
|
||
|
||
// LibraryReuseSources 读这个目标的持久来源(含已被恢复顶替的生效版本)。
|
||
// 同步来源要求同时存在 production_effective 版本登记:只写成功但版本没登记的
|
||
// 作业不算可审计来源。
|
||
func (s *Store) LibraryReuseSources(ctx context.Context, scope string, limit int) ([]LibraryReuseSource, error) {
|
||
if limit < 1 || limit > 200 {
|
||
limit = 100
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT j.id, j.target_snapshot_id, 'sync', coalesce(j.finished_at, j.updated_at)
|
||
FROM library_jobs j
|
||
WHERE j.scope=$1 AND j.target_id=$3 AND j.kind='sync' AND j.state='succeeded' AND coalesce(j.target_snapshot_id,'')<>''
|
||
AND EXISTS (SELECT 1 FROM library_versions v
|
||
WHERE v.scope=j.scope AND v.target_id=j.target_id AND v.kind='production_effective'
|
||
AND v.snapshot_id=j.target_snapshot_id AND v.source_job_id=j.id)
|
||
UNION ALL
|
||
SELECT coalesce(v.source_job_id,''), v.snapshot_id, 'prebackup', v.created_at
|
||
FROM library_versions v
|
||
WHERE v.scope=$1 AND v.target_id=$3 AND v.kind='production_prebackup' AND coalesce(v.snapshot_id,'')<>''
|
||
ORDER BY 4 DESC
|
||
LIMIT $2`, scope, limit, s.libraryTargetID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("load library reuse sources: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
sources := make([]LibraryReuseSource, 0, 8)
|
||
for rows.Next() {
|
||
var source LibraryReuseSource
|
||
if err := rows.Scan(&source.JobID, &source.SnapshotID, &source.Kind, &source.FinishedAt); err != nil {
|
||
return nil, fmt.Errorf("scan library reuse source: %w", err)
|
||
}
|
||
sources = append(sources, source)
|
||
}
|
||
return sources, rows.Err()
|
||
}
|
||
|
||
func (s *Store) libraryStages(ctx context.Context, id string) ([]governance.LibraryStage, error) {
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT stage, state, total, completed, bytes_total, bytes_done, detail, updated_at
|
||
FROM library_job_stages WHERE job_id=$1 ORDER BY updated_at, stage`, id)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("load library job stages: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
result := make([]governance.LibraryStage, 0, 7)
|
||
for rows.Next() {
|
||
var stage governance.LibraryStage
|
||
if err := rows.Scan(&stage.Stage, &stage.State, &stage.Total, &stage.Completed, &stage.BytesTotal,
|
||
&stage.BytesDone, &stage.Detail, &stage.UpdatedAt); err != nil {
|
||
return nil, fmt.Errorf("scan library job stage: %w", err)
|
||
}
|
||
result = append(result, stage)
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
// ListLibraryJobs 返回某目标最近的作业(按创建时间倒序),供页面在重启后恢复展示。
|
||
func (s *Store) ListLibraryJobs(ctx context.Context, scope string, limit int) ([]governance.LibraryJob, error) {
|
||
if limit < 1 || limit > 50 {
|
||
return nil, errors.New("library job list limit out of range")
|
||
}
|
||
rows, err := s.db.Query(ctx, `SELECT id FROM library_jobs WHERE scope=$1 AND (kind='seal' OR target_id=$3) ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit, s.libraryTargetID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list library jobs: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
ids := make([]string, 0, limit)
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil, fmt.Errorf("scan library job id: %w", err)
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
result := make([]governance.LibraryJob, 0, len(ids))
|
||
for _, id := range ids {
|
||
job, err := s.LibraryJob(ctx, id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result = append(result, job)
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// UpdateLibraryStage 只改调用方给了值的维度:nil 表示"这一维还没有可靠数据",
|
||
// 保持 NULL,绝不写成 0。
|
||
func (s *Store) UpdateLibraryStage(ctx context.Context, jobID, stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error {
|
||
tag, err := s.db.Exec(ctx, `
|
||
UPDATE library_job_stages SET
|
||
state = CASE WHEN $3='' THEN state ELSE $3 END,
|
||
total = COALESCE($4, total),
|
||
completed = COALESCE($5, completed),
|
||
bytes_total = COALESCE($6, bytes_total),
|
||
bytes_done = COALESCE($7, bytes_done),
|
||
detail = CASE WHEN $8='' THEN detail ELSE $8 END,
|
||
updated_at = $9
|
||
WHERE job_id=$1 AND stage=$2`, jobID, stage, state, total, completed, bytesTotal, bytesDone, detail, now.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("update library stage: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return errors.New("library stage not found")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// AttachLibraryQueueJob 把队列作业号写到已建的业务作业上(两步创建,避免先入队后才发现互斥冲突)。
|
||
func (s *Store) AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error {
|
||
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET queue_job_id=$2, updated_at=$3 WHERE id=$1`, jobID, queueJobID, now.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("attach library queue job: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return errors.New("library job not found")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *Store) CancelLibraryJob(ctx context.Context, jobID, reason string, now time.Time) error {
|
||
return s.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobCancelled, "", reason, "", "", "", now)
|
||
}
|
||
|
||
func (s *Store) AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error {
|
||
finished := any(nil)
|
||
if state == governance.LibraryJobSucceeded || state == governance.LibraryJobFailed || state == governance.LibraryJobCancelled {
|
||
finished = now.UTC()
|
||
}
|
||
tag, err := s.db.Exec(ctx, `
|
||
UPDATE library_jobs SET
|
||
state=$2,
|
||
stage = CASE WHEN $3='' THEN stage ELSE $3 END,
|
||
error = CASE WHEN $4='' THEN error ELSE $4 END,
|
||
recovery = CASE WHEN $5='' THEN recovery ELSE $5 END,
|
||
snapshot_id = CASE WHEN $6='' THEN snapshot_id ELSE $6 END,
|
||
target_snapshot_id = CASE WHEN $7='' THEN target_snapshot_id ELSE $7 END,
|
||
updated_at=$8,
|
||
finished_at=COALESCE($9, finished_at)
|
||
WHERE id=$1`, jobID, string(state), stage, message, recovery, snapshotID, targetSnapshotID, now.UTC(), finished)
|
||
if err != nil {
|
||
return fmt.Errorf("advance library job: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return errors.New("library job not found")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// RecordLibraryVersion 写入一条**不可变版本事件**,并在需要时推进 current 指针。
|
||
//
|
||
// 与旧实现的区别(F02/F09 的根因):
|
||
// - 事件 ID 含 operation:不同 operation 引用同一快照是两条事件,`source_job_id`/
|
||
// `evidence`/`created_at` 谁也不覆盖谁(旧实现是 `ON CONFLICT DO UPDATE` 整份替换)。
|
||
// - 同一 operation 重放是幂等:`ON CONFLICT (id) DO NOTHING`,已存行原样返回。
|
||
// - "谁当前生效"由 `library_current_versions` 表达,用单调次序做条件更新;
|
||
// 更早登记的事实不能把指针拉回历史,旧 epoch 在写边界就被守卫拒绝(见 guard)。
|
||
// - 生效/恢复事件才动指针;预备份/封存只追加事件。
|
||
func (s *Store) RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error {
|
||
targetID := s.libraryTargetID
|
||
if version.Kind == governance.LibraryVersionTestSealed {
|
||
targetID = ""
|
||
}
|
||
evidence, err := json.Marshal(version.Evidence)
|
||
if err != nil {
|
||
return fmt.Errorf("encode library version evidence: %w", err)
|
||
}
|
||
tx, err := s.db.Begin(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("begin library version: %w", err)
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
|
||
// 执行代次守卫:写事件的人必须还是当前代次(失租旧执行器不能登记"我成功了")。
|
||
if version.SourceJobID != "" && version.AttemptEpoch > 0 {
|
||
if err := guardLibraryJobEpoch(ctx, tx, version.SourceJobID, version.AttemptEpoch, version.AttemptOwner); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
|
||
// 事件身份 = (kind, scope, 快照, operation)。同一身份的重复登记只有两种合法形态:
|
||
// · 同一执行者补登记**新增**的字段(预备份先登记、写集合随后补上);
|
||
// · 完全没有新字段的纯重放。
|
||
// 所以冲突时只做**加法**:已有键永远赢(右操作数是已有 evidence),身份字段
|
||
// (source_job_id/operation_id/created_at)一格都不许改;换了执行者或换了意图的
|
||
// 登记在这里匹配不到行,什么也不写(F02/F09 不能靠 upsert 改写归属)。
|
||
// xmax = 0 才是真的新插入:重放同一份证据(或换了执行者被 WHERE 挡住)都不算新事实,
|
||
// 否则一次纯重放就能把 current 再"推进"一次,把后来的操作顶掉。
|
||
var inserted bool
|
||
err = tx.QueryRow(ctx, `
|
||
INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id,
|
||
operation_id, attempt_epoch, evidence, created_by, created_at, target_id)
|
||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)
|
||
ON CONFLICT (id) DO UPDATE
|
||
SET evidence = EXCLUDED.evidence || library_versions.evidence
|
||
WHERE library_versions.source_job_id = EXCLUDED.source_job_id
|
||
AND library_versions.operation_id = EXCLUDED.operation_id
|
||
AND library_versions.target_id = EXCLUDED.target_id
|
||
RETURNING (xmax = 0)`,
|
||
version.ID, version.Scope, string(version.Kind), version.SnapshotID, version.VersionLabel,
|
||
version.SourceJobID, version.OperationID, version.AttemptEpoch, evidence, version.CreatedBy,
|
||
version.CreatedAt.UTC(), targetID).Scan(&inserted)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
var existingTarget string
|
||
if loadErr := tx.QueryRow(ctx, `SELECT target_id FROM library_versions WHERE id=$1`, version.ID).Scan(&existingTarget); loadErr != nil {
|
||
return loadErr
|
||
}
|
||
if existingTarget != targetID {
|
||
return errors.New("library version identity belongs to another target")
|
||
}
|
||
// 同一身份已经存在、但归属是别的执行者/别的意图:不覆盖、不报错,也不推进 current。
|
||
inserted = false
|
||
err = nil
|
||
}
|
||
if err != nil {
|
||
return fmt.Errorf("record library version event: %w", err)
|
||
}
|
||
|
||
if version.Kind == governance.LibraryVersionProductionEffective || version.Kind == governance.LibraryVersionRecoveryVerified {
|
||
if err := advanceLibraryCurrent(ctx, tx, version, inserted, targetID); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return tx.Commit(ctx)
|
||
}
|
||
|
||
// advanceLibraryCurrent 用单调次序条件推进 current 指针。
|
||
//
|
||
// 次序键 = (意图创建时间 | operation | epoch),**不是**事件登记时间(父的复核项):
|
||
// 拿登记时间排序,一次迟到的旧 operation 重放(例如重跑几天前的同步)就会因为"我刚登记"
|
||
// 而把 current 推回历史。意图的创建时间不会因为重放而变新,所以旧意图永远推不动
|
||
// 比它更新的意图。
|
||
//
|
||
// 恢复要不要能把 current 指回更早的**内容**?要,但它是通过**新的意图**(kind=recover,
|
||
// 创建于恢复时刻)做到的:内容更旧、意图更新,两者不矛盾。旧代次能不能写由 epoch
|
||
// 守卫回答(见 libraryFence),不由时间回答。
|
||
func advanceLibraryCurrent(ctx context.Context, tx pgx.Tx, version governance.LibraryVersion, inserted bool, targetID string) error {
|
||
orderAt := version.CreatedAt
|
||
if version.OperationID != "" {
|
||
var created *time.Time
|
||
err := tx.QueryRow(ctx, `SELECT created_at FROM library_operations WHERE id=$1`,
|
||
version.OperationID).Scan(&created)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return fmt.Errorf("事件引用的 operation %s 不存在:拒绝推进 current", version.OperationID)
|
||
}
|
||
if err != nil {
|
||
return fmt.Errorf("load library operation for current ordering: %w", err)
|
||
}
|
||
if created != nil {
|
||
orderAt = *created
|
||
}
|
||
}
|
||
orderKey := governance.CurrentOrderKey(orderAt, version.OperationID, version.AttemptEpoch)
|
||
if !inserted {
|
||
// 重放:事件已经在台账里。确认指针没有被别的更晚事实覆盖,否则只确认自己的历史。
|
||
var currentID string
|
||
err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1 AND target_id=$2 FOR UPDATE`,
|
||
version.Scope, targetID).Scan(¤tID)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return nil
|
||
}
|
||
if err != nil {
|
||
return fmt.Errorf("load library current pointer: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
tag, err := tx.Exec(ctx, `
|
||
INSERT INTO library_current_versions (scope, version_id, snapshot_id, operation_id, attempt_epoch, order_key, updated_at, target_id)
|
||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
|
||
ON CONFLICT (scope, target_id) DO UPDATE SET
|
||
version_id=EXCLUDED.version_id, snapshot_id=EXCLUDED.snapshot_id,
|
||
operation_id=EXCLUDED.operation_id, attempt_epoch=EXCLUDED.attempt_epoch,
|
||
order_key=EXCLUDED.order_key, updated_at=EXCLUDED.updated_at
|
||
WHERE EXCLUDED.order_key > library_current_versions.order_key`,
|
||
version.Scope, version.ID, version.SnapshotID, version.OperationID, version.AttemptEpoch, orderKey,
|
||
version.CreatedAt.UTC(), targetID)
|
||
if err != nil {
|
||
return fmt.Errorf("advance library current pointer: %w", err)
|
||
}
|
||
var currentID string
|
||
if err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1 AND target_id=$2`,
|
||
version.Scope, targetID).Scan(¤tID); err != nil {
|
||
return fmt.Errorf("read back library current pointer: %w", err)
|
||
}
|
||
if tag.RowsAffected() == 0 && currentID != version.ID {
|
||
return fmt.Errorf("%w(scope=%s:当前 %s,本条 %s)", governance.ErrLibraryCurrentStale,
|
||
version.Scope, currentID, version.ID)
|
||
}
|
||
// superseded_at 只是展示元数据("这条已被顶替"),不改任何证据字段。
|
||
// 它跟着指针走:指针指向谁,谁就是未顶替的那一行。
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE library_versions SET superseded_at = CASE WHEN id=$2 THEN NULL ELSE COALESCE(superseded_at, $3) END
|
||
WHERE scope=$1 AND target_id=$4 AND kind IN ('production_effective','recovery_verified')`, version.Scope, currentID,
|
||
version.CreatedAt.UTC(), targetID); err != nil {
|
||
return fmt.Errorf("refresh library version supersede metadata: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// InterruptedLibraryJobs 找出"业务作业 running、但队列里已没有活跃作业"的作业:
|
||
// worker 被杀/重启后它们需要被接管,而不是被当成失败丢掉已完成的预备份。
|
||
func (s *Store) InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error) {
|
||
// lease 过期才算中断:worker 刚被杀但 lease 未到期时,队列里还是 running,
|
||
// 直接判"没中断"会让作业永远悬着;所以这里按 lease_until 判断。
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT lj.id FROM library_jobs lj
|
||
LEFT JOIN jobs j ON j.id=lj.queue_job_id
|
||
WHERE lj.kind IN ('sync','restore','baseline')
|
||
AND (
|
||
-- 业务作业在跑、队列已经没了/停了/租约过期
|
||
(lj.state='running' AND (j.id IS NULL OR j.state IN ('failed','completed','cancelled','dead')
|
||
OR (j.state IN ('pending','running','retry') AND j.lease_until IS NOT NULL AND j.lease_until < now())))
|
||
-- 业务作业已被前一轮接管置为 pending,但队列作业的租约也没了(worker 又死了)
|
||
OR (lj.state='pending' AND j.state='running' AND j.lease_until IS NOT NULL AND j.lease_until < now())
|
||
)
|
||
AND lj.target_id=$1
|
||
ORDER BY lj.created_at`, s.libraryTargetID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list interrupted library jobs: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
ids := make([]string, 0, 4)
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil, err
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
jobs := make([]governance.LibraryJob, 0, len(ids))
|
||
for _, id := range ids {
|
||
job, err := s.LibraryJob(ctx, id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
jobs = append(jobs, job)
|
||
}
|
||
return jobs, nil
|
||
}
|
||
|
||
// PrepareLibraryJobResume 把**同一个业务作业**放回 pending,供续作/接管重新执行。
|
||
//
|
||
// 为什么是原作业而不是新开一行(R3):维护窗口 owner_job_id、freeze journal 文件名、
|
||
// 预备份与写集合的身份全部按业务作业 id 落在持久层。新开 `-resume` 作业只能继承
|
||
// operation_id,其它所有者都还指向旧作业——AcquireInstanceMaintenance 会拒绝它,
|
||
// journal/预备份也查不到,于是"续作"只是一个漂亮的新行。
|
||
//
|
||
// 条件更新:只有 failed/cancelled 才允许置回 pending;终态时间与错误清零,
|
||
// 旧 attempt 行保留(历史不可变)。代次由 BeginLibraryAttempt 在 worker 起手时递增。
|
||
func (s *Store) PrepareLibraryJobResume(ctx context.Context, jobID string, now time.Time) (governance.LibraryJob, bool, error) {
|
||
if _, err := s.LibraryJob(ctx, jobID); err != nil {
|
||
return governance.LibraryJob{}, false, err
|
||
}
|
||
tx, err := s.db.Begin(ctx)
|
||
if err != nil {
|
||
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
// 续作是人工发起的接管:作业已经 failed/cancelled,它那一代执行者不可能还在合法地写。
|
||
// 必须**显式结束**那一代(行保留、标 abandoned),否则它的租约还没过期,
|
||
// BeginLibraryAttempt 会认为"有活跃执行者"而拒绝新代次——续作就永远开不起来。
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE library_attempts SET state='abandoned', finished_at=$2, outcome='superseded by operator resume',
|
||
lease_until=NULL
|
||
WHERE job_id=$1 AND state IN ('claimed','running')`, jobID, now.UTC()); err != nil {
|
||
return governance.LibraryJob{}, false, fmt.Errorf("retire attempts for resume: %w", err)
|
||
}
|
||
tag, err := tx.Exec(ctx, `
|
||
UPDATE library_jobs SET state='pending', error='', recovery='', finished_at=NULL, updated_at=$2,
|
||
lease_owner='', lease_until=NULL
|
||
WHERE id=$1 AND state IN ('failed','cancelled')`, jobID, now.UTC())
|
||
if err != nil {
|
||
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return governance.LibraryJob{}, false, fmt.Errorf("prepare library job resume: %w", err)
|
||
}
|
||
job, loadErr := s.LibraryJob(ctx, jobID)
|
||
if loadErr != nil {
|
||
return governance.LibraryJob{}, false, loadErr
|
||
}
|
||
return job, tag.RowsAffected() == 1, nil
|
||
}
|
||
|
||
// ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零,
|
||
// 否则 attempt<max_attempts 永远不成立,作业再也不会被任何 worker 领取。
|
||
//
|
||
// 但**不能清掉仍然有效的租约**:那等于把正在执行的那一代直接踢掉,而它可能已经在写目标
|
||
// (F13 的原始形态)。只有租约为空或已过期才允许重置。
|
||
func (s *Store) ResetQueueJobForResume(ctx context.Context, queueJobID string, now time.Time) error {
|
||
tag, err := s.db.Exec(ctx, `UPDATE jobs
|
||
SET state='pending', attempt=0, lease_owner=NULL, lease_until=NULL, available_at=$2, updated_at=$2
|
||
WHERE id=$1 AND state IN ('running','retry','failed','pending')
|
||
AND (lease_until IS NULL OR lease_until < $2)`, queueJobID, now.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("reset queue job for resume: %w", err)
|
||
}
|
||
if tag.RowsAffected() == 0 {
|
||
// 有活跃租约:另一个执行者正在跑,本次接管必须让路(调用方跳过即可)。
|
||
return ErrLibraryAttemptHeld
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ClaimLibraryJobForResume 用一次条件更新把中断作业抢过来:只有把 running 改成 pending
|
||
// 的那一个 worker 拿到 true,多个 worker 同时重启也只会有一个执行(fencing)。
|
||
//
|
||
// 条件里必须看租约:pending→pending 的无条件认领会让两个 worker 都"认领成功",
|
||
// 然后各自跑一遍同一个窗口(F13)。租约仍有效时直接失败,让调用方跳过。
|
||
func (s *Store) ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error) {
|
||
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET state='pending', updated_at=$2
|
||
WHERE id=$1 AND target_id=$3 AND state IN ('running','pending')
|
||
AND (lease_owner='' OR lease_until IS NULL OR lease_until < $2)`, jobID, now.UTC(), s.libraryTargetID)
|
||
if err != nil {
|
||
return false, fmt.Errorf("claim interrupted library job: %w", err)
|
||
}
|
||
return tag.RowsAffected() == 1, nil
|
||
}
|
||
|
||
// ReconcileLibraryJobs 收拾"业务作业还在飞、队列里已经没有对应作业"的孤儿:
|
||
// 它们会永久占用 (scope,kind) 唯一索引,让同一目标再也发不出作业。
|
||
// 只标失败并写明恢复责任,不猜结果、不重放。
|
||
//
|
||
// kinds 是**本进程负责的那几类**:两台执行器(本机封存 / 云端发布)各自收拾自己的
|
||
// 作业,跨域"顺手收拾"会把别人的在飞作业标失败,并把恢复责任写错(封存要 agent
|
||
// 恢复来源,发布要按预备份确认目标)。kinds 为空直接报错,不允许退化成"收拾全部"。
|
||
func (s *Store) ReconcileLibraryJobs(ctx context.Context, now time.Time, kinds []string) (int, error) {
|
||
if len(kinds) == 0 {
|
||
return 0, errors.New("reconcile library jobs requires the kinds this executor owns")
|
||
}
|
||
if _, err := s.db.Exec(ctx, `
|
||
UPDATE library_jobs AS lj
|
||
SET state='failed', error='队列中已无对应作业,业务作业未收到结果',
|
||
recovery=CASE WHEN lj.kind IN ('sync','restore','recover','baseline')
|
||
THEN '未确认目标状态:若已进入维护窗口,按预备份恢复并核验后再重发'
|
||
ELSE '封存作业由 agent 恢复来源;确认来源 healthy 后可重发' END,
|
||
updated_at=$1, finished_at=$1
|
||
WHERE lj.state IN ('pending','running')
|
||
AND lj.kind = ANY($2)
|
||
AND (lj.kind='seal' OR lj.target_id=$3)
|
||
AND lj.queue_job_id <> ''
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM jobs AS j
|
||
WHERE j.id = lj.queue_job_id AND j.state IN ('pending','running','retry')
|
||
)`, now.UTC(), kinds, s.libraryTargetID); err != nil {
|
||
return 0, fmt.Errorf("reconcile library jobs: %w", err)
|
||
}
|
||
// 建了业务作业但队列投递没完成(queue_job_id 为空)超过 5 分钟,也是孤儿。
|
||
tag, err := s.db.Exec(ctx, `
|
||
UPDATE library_jobs
|
||
SET state='failed', error='队列投递未完成,作业从未开始', recovery='无需恢复来源;可直接重发', updated_at=$1, finished_at=$1
|
||
WHERE state IN ('pending','running') AND queue_job_id='' AND kind = ANY($3) AND (kind='seal' OR target_id=$4) AND created_at < $2`,
|
||
now.UTC(), now.Add(-5*time.Minute).UTC(), kinds, s.libraryTargetID)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("reconcile unattached library jobs: %w", err)
|
||
}
|
||
return int(tag.RowsAffected()), nil
|
||
}
|
||
|
||
// LibraryVersionByRef 按 (scope, kind, 作业, 快照) 精确定位一条版本记录。
|
||
//
|
||
// 为什么不能"取最近 N 条再在内存里筛":那是窗口,历史一旦超过窗口上限,
|
||
// 登记/接管/恢复就会把"存在"读成"不存在"——真实现场是写集合的引用无法登记,
|
||
// 报"读不到现有版本证据 / library version limit out of range"。
|
||
// 身份查询必须走索引条件,不许依赖排序窗口(把上限从 200 改成 100 只是换个数)。
|
||
//
|
||
// 同一 (scope, kind, snapshot) 只该有一行(id 由这三者决定,写入是 upsert)。
|
||
// 命中多行、或命中的那行归别的作业所有,都说明台账不唯一:返回错误让调用方
|
||
// 拒绝,不许挑一条"看起来对"的记录去恢复生产。
|
||
func (s *Store) LibraryVersionByRef(ctx context.Context, scope string, kind governance.LibraryVersionKind,
|
||
sourceJobID, snapshotID string) (governance.LibraryVersion, bool, error) {
|
||
if scope == "" || kind == "" || sourceJobID == "" || snapshotID == "" {
|
||
return governance.LibraryVersion{}, false, errors.New("library version ref requires scope, kind, job and snapshot")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at
|
||
FROM library_versions
|
||
WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4 AND (kind='test_sealed' OR target_id=$5)
|
||
ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), sourceJobID, snapshotID, s.libraryTargetID)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("load library version by ref: %w", err)
|
||
}
|
||
items, err := scanLibraryVersions(rows, 2)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, err
|
||
}
|
||
if len(items) > 1 {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("同一身份存在 %d 条版本记录(作业 %s,快照 %s):拒绝挑选",
|
||
len(items), sourceJobID, snapshotID)
|
||
}
|
||
if len(items) == 1 {
|
||
return items[0], true, nil
|
||
}
|
||
// 没命中时要区分"确实没有"与"这一行归别的作业":后者是身份冲突,
|
||
// 当成"没有记录"就会允许调用方覆盖别人的台账。
|
||
var owner string
|
||
err = s.db.QueryRow(ctx, `
|
||
SELECT source_job_id FROM library_versions WHERE scope=$1 AND kind=$2 AND snapshot_id=$3 AND (kind='test_sealed' OR target_id=$4)
|
||
ORDER BY created_at DESC LIMIT 1`, scope, string(kind), snapshotID, s.libraryTargetID).Scan(&owner)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryVersion{}, false, nil
|
||
}
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("load library version owner: %w", err)
|
||
}
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("版本记录(快照 %s)属于作业 %s,不是 %s:拒绝借用",
|
||
snapshotID, owner, sourceJobID)
|
||
}
|
||
|
||
// scanLibraryVersions 是版本行唯一的扫描实现:两个查询(窗口列表与按身份精确查)
|
||
// 必须对同一种形状解码,否则"查得到"和"读得对"会分叉。
|
||
func scanLibraryVersions(rows pgx.Rows, capacity int) ([]governance.LibraryVersion, error) {
|
||
defer rows.Close()
|
||
result := make([]governance.LibraryVersion, 0, capacity)
|
||
for rows.Next() {
|
||
var item governance.LibraryVersion
|
||
var kind string
|
||
var evidence []byte
|
||
if err := rows.Scan(&item.ID, &item.Scope, &kind, &item.SnapshotID, &item.VersionLabel, &item.SourceJobID,
|
||
&item.OperationID, &item.AttemptEpoch, &evidence, &item.CreatedBy, &item.CreatedAt, &item.SupersededAt); err != nil {
|
||
return nil, fmt.Errorf("scan library version: %w", err)
|
||
}
|
||
item.Kind = governance.LibraryVersionKind(kind)
|
||
if len(evidence) > 0 {
|
||
_ = json.Unmarshal(evidence, &item.Evidence)
|
||
}
|
||
result = append(result, item)
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
// LibraryVersions 是**分页展示**用的窗口列表(1..100),语义是"最近 N 条"。
|
||
// 任何"当前生效 / 本快照是否生效过"的判断都不许用它:窗口一满就会把存在读成不存在。
|
||
func (s *Store) LibraryVersions(ctx context.Context, scope string, limit int) ([]governance.LibraryVersion, error) {
|
||
if limit < 1 || limit > 100 {
|
||
return nil, errors.New("library version limit out of range")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at
|
||
FROM library_versions WHERE scope=$1 AND (kind='test_sealed' OR target_id=$3) ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit, s.libraryTargetID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list library versions: %w", err)
|
||
}
|
||
return scanLibraryVersions(rows, limit)
|
||
}
|
||
|
||
// LibraryCurrentVersion 返回这个 scope **当前生效**的事实(由指针表回答)。
|
||
//
|
||
// 三个理由必须一起满足:
|
||
//
|
||
// · 不能用窗口列表——生效事实可能很久以前登记,之后写进来的预备份/封存会把它挤出
|
||
// 任何"最近 N 条",接管/基线/闸口就会把"已生效"读成"没有生效"。
|
||
// · 不能靠扫 library_versions.kind='production_effective' 挑未顶替行——事件不可变,
|
||
// "谁是当前"是指针的事实,不是事件行的可变字段(F02)。
|
||
// · 指针每 (scope, target_id) 至多一行,历史归属不明的空身份不归新目标。
|
||
//
|
||
// 指针可能指向 production_effective,也可能指向 recovery_verified(按预备份恢复后
|
||
// 的当前内容身份)——调用方只能用它回答"当前是什么内容",不能假设它是一次成功发布。
|
||
func (s *Store) LibraryCurrentVersion(ctx context.Context, scope string) (governance.LibraryVersion, bool, error) {
|
||
if scope == "" {
|
||
return governance.LibraryVersion{}, false, errors.New("library current version requires scope")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT v.id, v.scope, v.kind, v.snapshot_id, v.version_label, v.source_job_id, v.operation_id, v.attempt_epoch,
|
||
v.evidence, v.created_by, v.created_at, v.superseded_at
|
||
FROM library_current_versions c JOIN library_versions v ON v.id = c.version_id
|
||
WHERE c.scope=$1 AND c.target_id=$2 AND v.target_id=c.target_id`, scope, s.libraryTargetID)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("load current library version: %w", err)
|
||
}
|
||
items, err := scanLibraryVersions(rows, 2)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, err
|
||
}
|
||
if len(items) > 1 {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("scope=%s 的当前指针指向 %d 行事件:台账被写坏,拒绝挑选",
|
||
scope, len(items))
|
||
}
|
||
if len(items) == 1 {
|
||
return items[0], true, nil
|
||
}
|
||
return governance.LibraryVersion{}, false, nil
|
||
}
|
||
|
||
// LibraryVersionOfSnapshot 按 (scope, kind, 快照) 取这个快照**自己的**登记记录(不限作业)。
|
||
//
|
||
// 用途是"本快照曾经生效过吗/是否已被顶替":版本 id 由 (kind, scope, 快照) 决定,
|
||
// 一个快照在一种类型下只该有一行,作业归属会随恢复/重放被改写,所以不能用作业当条件。
|
||
// 命中多行同样拒绝挑选。
|
||
func (s *Store) LibraryVersionOfSnapshot(ctx context.Context, scope string,
|
||
kind governance.LibraryVersionKind, snapshotID string) (governance.LibraryVersion, bool, error) {
|
||
if scope == "" || kind == "" || snapshotID == "" {
|
||
return governance.LibraryVersion{}, false, errors.New("library version snapshot requires scope, kind and snapshot")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at
|
||
FROM library_versions
|
||
WHERE scope=$1 AND kind=$2 AND snapshot_id=$3 AND (kind='test_sealed' OR target_id=$4)
|
||
ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), snapshotID, s.libraryTargetID)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("load library version of snapshot: %w", err)
|
||
}
|
||
found, err := scanLibraryVersions(rows, 2)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, err
|
||
}
|
||
// 同一快照可以被多次意图引用,每次 operation 一条事件,这是**正常历史**;
|
||
// 但"两条同身份的行"是台账被写坏(或迁移前没有 operation 身份的未知证据),
|
||
// 这时挑选任何一条都是替调用方做决定,必须拒绝(未知证据 fail closed)。
|
||
if len(found) == 1 {
|
||
return found[0], true, nil
|
||
}
|
||
if len(found) >= 2 {
|
||
if found[0].OperationID == "" || found[1].OperationID == "" {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf(
|
||
"快照 %s 在 %s 下有 %d 条无执行身份的记录:拒绝挑选", snapshotID, kind, len(found))
|
||
}
|
||
if found[0].OperationID == found[1].OperationID {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf(
|
||
"快照 %s 在 %s 下同一意图登记了多行:拒绝挑选", snapshotID, kind)
|
||
}
|
||
return found[0], true, nil
|
||
}
|
||
return governance.LibraryVersion{}, false, nil
|
||
}
|
||
|
||
// CreateLibraryAuthorization 记录一次生产发布授权;同一 (scope,snapshot,digest) 只保留最新有效。
|
||
func (s *Store) CreateLibraryAuthorization(ctx context.Context, item governance.LibraryReleaseAuthorization) error {
|
||
if item.ID == "" || item.Scope == "" || item.SnapshotID == "" || item.ParamsDigest == "" || item.ExpiresAt.IsZero() {
|
||
return errors.New("invalid library release authorization")
|
||
}
|
||
_, err := s.db.Exec(ctx, `
|
||
INSERT INTO library_release_authorizations (id, scope, snapshot_id, params_digest, authorized_by, expires_at, created_at, target_id)
|
||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`,
|
||
item.ID, item.Scope, item.SnapshotID, item.ParamsDigest, item.AuthorizedBy, item.ExpiresAt.UTC(), item.CreatedAt.UTC(), s.libraryTargetID)
|
||
if err != nil {
|
||
return fmt.Errorf("create library release authorization: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ActiveLibraryAuthorization 只返回"未撤销、未过期、且参数摘要完全一致"的授权。
|
||
// 参数一变就查不到,自然拒绝,而不是靠调用方记得复核。
|
||
func (s *Store) ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, paramsDigest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error) {
|
||
var item governance.LibraryReleaseAuthorization
|
||
err := s.db.QueryRow(ctx, `
|
||
SELECT id, scope, snapshot_id, params_digest, authorized_by, expires_at, revoked_at, created_at
|
||
FROM library_release_authorizations
|
||
WHERE scope=$1 AND snapshot_id=$2 AND params_digest=$3 AND target_id=$5 AND revoked_at IS NULL AND expires_at > $4
|
||
ORDER BY expires_at DESC, created_at DESC
|
||
LIMIT 1`, scope, snapshotID, paramsDigest, now.UTC(), s.libraryTargetID).Scan(&item.ID, &item.Scope, &item.SnapshotID,
|
||
&item.ParamsDigest, &item.AuthorizedBy, &item.ExpiresAt, &item.RevokedAt, &item.CreatedAt)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryReleaseAuthorization{}, false, nil
|
||
}
|
||
if err != nil {
|
||
return governance.LibraryReleaseAuthorization{}, false, fmt.Errorf("load library authorization: %w", err)
|
||
}
|
||
return item, true, nil
|
||
}
|
||
|
||
func (s *Store) RevokeLibraryAuthorization(ctx context.Context, id string, now time.Time) error {
|
||
tag, err := s.db.Exec(ctx, `UPDATE library_release_authorizations SET revoked_at=$2 WHERE id=$1 AND target_id=$3 AND revoked_at IS NULL`, id, now.UTC(), s.libraryTargetID)
|
||
if err != nil {
|
||
return fmt.Errorf("revoke library authorization: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return errors.New("library authorization not found or already revoked")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *Store) LibraryAuthorizations(ctx context.Context, scope string, limit int) ([]governance.LibraryReleaseAuthorization, error) {
|
||
if limit < 1 || limit > 100 {
|
||
return nil, errors.New("library authorization limit out of range")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT id, scope, snapshot_id, params_digest, authorized_by, expires_at, revoked_at, created_at
|
||
FROM library_release_authorizations WHERE scope=$1 AND target_id=$3 ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit, s.libraryTargetID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list library authorizations: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
result := make([]governance.LibraryReleaseAuthorization, 0, limit)
|
||
for rows.Next() {
|
||
var item governance.LibraryReleaseAuthorization
|
||
if err := rows.Scan(&item.ID, &item.Scope, &item.SnapshotID, &item.ParamsDigest, &item.AuthorizedBy,
|
||
&item.ExpiresAt, &item.RevokedAt, &item.CreatedAt); err != nil {
|
||
return nil, fmt.Errorf("scan library authorization: %w", err)
|
||
}
|
||
result = append(result, item)
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
// LibraryVersionsBySnapshot 返回这份快照在该 scope 下的全部版本事件(含证据)。
|
||
//
|
||
// 原产地不能只看 kind:目标权威基线也是 production_effective,但它是**目标侧的真实捕获**
|
||
// (证据里 action=target_baseline_init / origin=target_baseline)。所以判定必须读到证据,
|
||
// 不能把"是 effective"直接当成"来自发布"或"来自目标"。
|
||
func (s *Store) LibraryVersionsBySnapshot(ctx context.Context, scope, snapshotID string) ([]governance.LibraryVersion, error) {
|
||
if scope == "" || snapshotID == "" {
|
||
return nil, errors.New("snapshot versions requires scope and snapshot")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT id, scope, kind, snapshot_id, source_job_id, COALESCE(evidence, '{}'::jsonb), created_at
|
||
FROM library_versions WHERE scope=$1 AND snapshot_id=$2 AND (kind='test_sealed' OR target_id=$3) ORDER BY created_at`, scope, snapshotID, s.libraryTargetID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("read snapshot versions: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
items := make([]governance.LibraryVersion, 0, 4)
|
||
for rows.Next() {
|
||
var item governance.LibraryVersion
|
||
var kind string
|
||
var evidence []byte
|
||
if err := rows.Scan(&item.ID, &item.Scope, &kind, &item.SnapshotID, &item.SourceJobID, &evidence, &item.CreatedAt); err != nil {
|
||
return nil, err
|
||
}
|
||
item.Evidence = decodeEvidenceObject(evidence)
|
||
item.Kind = governance.LibraryVersionKind(kind)
|
||
items = append(items, item)
|
||
}
|
||
return items, rows.Err()
|
||
}
|
||
|
||
// decodeEvidenceObject 把证据列解成对象:历史行里也有把证据存成"数组"的
|
||
// (早期任务把一组事件直接塞进 jsonb)。读原产地只关心字段,所以数组取最后一个对象,
|
||
// 解不动就返回 nil(调用方按"没有证据"处理,不会因此放宽判定)。
|
||
func decodeEvidenceObject(raw []byte) map[string]any {
|
||
trimmed := bytes.TrimSpace(raw)
|
||
if len(trimmed) == 0 {
|
||
return nil
|
||
}
|
||
if trimmed[0] == '{' {
|
||
var object map[string]any
|
||
if err := json.Unmarshal(trimmed, &object); err == nil {
|
||
return object
|
||
}
|
||
return nil
|
||
}
|
||
if trimmed[0] == '[' {
|
||
var items []map[string]any
|
||
if err := json.Unmarshal(trimmed, &items); err == nil && len(items) > 0 {
|
||
return items[len(items)-1]
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// LibrarySnapshotKinds 返回这份快照在该 scope 下登记过的版本类型。
|
||
//
|
||
// 用途是判定"这份快照的清单身份应该等于谁":封存版本(test_sealed)的清单写的是
|
||
// **源**的 tenant/KB;预备份/生效/恢复版本写的是**目标**的 tenant/KB。没有登记过
|
||
// 就说明来源不明,调用方必须拒绝搬运(不能信快照自报)。
|
||
func (s *Store) LibrarySnapshotKinds(ctx context.Context, scope, snapshotID string) ([]string, error) {
|
||
if scope == "" || snapshotID == "" {
|
||
return nil, errors.New("snapshot kinds requires scope and snapshot")
|
||
}
|
||
rows, err := s.db.Query(ctx, `SELECT DISTINCT kind FROM library_versions WHERE scope=$1 AND snapshot_id=$2 AND (kind='test_sealed' OR target_id=$3)`, scope, snapshotID, s.libraryTargetID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("read snapshot provenance: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
kinds := make([]string, 0, 4)
|
||
for rows.Next() {
|
||
var kind string
|
||
if err := rows.Scan(&kind); err != nil {
|
||
return nil, err
|
||
}
|
||
kinds = append(kinds, kind)
|
||
}
|
||
return kinds, rows.Err()
|
||
}
|
||
|
||
// SnapshotCandidate 是一份"本机可能需要有"的快照,以及它最近一次被引用的时间。
|
||
// CreatedAt 同时充当分页游标:按 (created_at, snapshot_id) 倒序,跨页不重不漏。
|
||
type SnapshotCandidate struct {
|
||
SnapshotID string
|
||
CreatedAt time.Time
|
||
}
|
||
|
||
// LibrarySnapshotCandidates 枚举该 scope 下**被引用过**的快照(分页,游标不是 OFFSET)。
|
||
//
|
||
// 为什么不能"只取最近 N 条成功作业的 snapshot_id":那会把回传永久卡在最早的 50 条上,
|
||
// 第 51 条以后的预备份、以及失败/待恢复作业留下的**恢复所需**预备份再也读不回来。
|
||
// 所以这里合并三处权威引用:
|
||
// - library_versions.snapshot_id(版本台账,含封存/预备份/生效/恢复);
|
||
// - library_jobs.snapshot_id(作业自己留下的预备份,含失败与 needs_restore);
|
||
// - library_jobs.target_snapshot_id(发布/恢复的输入快照,历史恢复要靠它寻址)。
|
||
//
|
||
// 游标语义:afterCreated/afterID 为空表示第一页;返回值不足 limit 即到底。
|
||
func (s *Store) LibrarySnapshotCandidates(ctx context.Context, scope string, afterCreated time.Time,
|
||
afterID string, limit int) ([]SnapshotCandidate, error) {
|
||
if scope == "" || limit <= 0 || limit > 500 {
|
||
return nil, errors.New("snapshot candidates requires scope and a bounded limit")
|
||
}
|
||
var cursor any
|
||
if !afterCreated.IsZero() {
|
||
cursor = afterCreated.UTC()
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
WITH referenced AS (
|
||
SELECT snapshot_id, created_at FROM library_versions WHERE scope=$1 AND (kind='test_sealed' OR target_id=$5) AND snapshot_id <> ''
|
||
UNION ALL
|
||
SELECT snapshot_id, created_at FROM library_jobs WHERE scope=$1 AND (kind='seal' OR target_id=$5) AND snapshot_id <> ''
|
||
UNION ALL
|
||
SELECT target_snapshot_id, created_at FROM library_jobs WHERE scope=$1 AND (kind='seal' OR target_id=$5) AND target_snapshot_id <> ''
|
||
), grouped AS (
|
||
SELECT snapshot_id, MAX(created_at) AS created_at FROM referenced GROUP BY snapshot_id
|
||
)
|
||
SELECT snapshot_id, created_at FROM grouped
|
||
WHERE $2::timestamptz IS NULL
|
||
OR created_at < $2::timestamptz
|
||
OR (created_at = $2::timestamptz AND snapshot_id < $3)
|
||
ORDER BY created_at DESC, snapshot_id DESC
|
||
LIMIT $4`, scope, cursor, afterID, limit, s.libraryTargetID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list snapshot candidates: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
items := make([]SnapshotCandidate, 0, limit)
|
||
for rows.Next() {
|
||
var item SnapshotCandidate
|
||
if err := rows.Scan(&item.SnapshotID, &item.CreatedAt); err != nil {
|
||
return nil, err
|
||
}
|
||
items = append(items, item)
|
||
}
|
||
return items, rows.Err()
|
||
}
|
||
|
||
// handoff 阶段的三个阶段(running/succeeded/failed)与业务作业行的所有权判定
|
||
// **必须在同一个事务里**:分开做时,读到的 owner 与自己写时已经不是同一个执行者,
|
||
// 迟到的旧 handoff 就能覆盖已经交给 release 的状态与阶段。
|
||
//
|
||
// 所有权只由 library_jobs.queue_job_id 决定(阶段行是否存在与所有权无关),
|
||
// 并且每次都在 `SELECT ... FOR UPDATE` 之后判定,跨表并发交接因此不能穿插。
|
||
|
||
func (s *Store) lockLibraryJob(ctx context.Context, tx pgx.Tx, libraryJobID, ownerQueueID string) (string, bool, error) {
|
||
var state string
|
||
var attached *string
|
||
if err := tx.QueryRow(ctx, `SELECT state, queue_job_id FROM library_jobs WHERE id=$1 FOR UPDATE`,
|
||
libraryJobID).Scan(&state, &attached); err != nil {
|
||
return "", false, fmt.Errorf("lock library job: %w", err)
|
||
}
|
||
current := ""
|
||
if attached != nil {
|
||
current = *attached
|
||
}
|
||
if current != ownerQueueID || (state != "pending" && state != "running") {
|
||
return state, false, nil
|
||
}
|
||
return state, true, nil
|
||
}
|
||
|
||
func upsertLibraryStage(ctx context.Context, tx pgx.Tx, libraryJobID, stage, state, detail string, now time.Time) error {
|
||
_, err := tx.Exec(ctx, `
|
||
INSERT INTO library_job_stages (job_id, stage, state, detail, updated_at)
|
||
VALUES ($1, $2, $3, $4, $5)
|
||
ON CONFLICT (job_id, stage) DO UPDATE
|
||
SET state=EXCLUDED.state, detail=EXCLUDED.detail, updated_at=EXCLUDED.updated_at`,
|
||
libraryJobID, stage, state, detail, now.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("record library stage %s: %w", stage, err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// BeginLibraryHandoff 在事务里创建/推进 handoff 阶段为 running,并确认所有权仍在。
|
||
// 返回 false 表示"这份作业已经不归本次 handoff 了",调用方不得再碰任何状态。
|
||
func (s *Store) BeginLibraryHandoff(ctx context.Context, libraryJobID, ownerQueueID, detail string,
|
||
now time.Time) (bool, error) {
|
||
tx, err := s.db.Begin(ctx)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
if _, owned, err := s.lockLibraryJob(ctx, tx, libraryJobID, ownerQueueID); err != nil {
|
||
return false, err
|
||
} else if !owned {
|
||
return false, nil
|
||
}
|
||
if err := upsertLibraryStage(ctx, tx, libraryJobID, "handoff", "running", detail, now); err != nil {
|
||
return false, err
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return false, err
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
// CompleteLibraryHandoff 在**一个事务**里:确认所有权 → 投递发布队列作业 → 把业务作业
|
||
// 交接给该队列作业 → 把 handoff 阶段写成 succeeded。任何一步失败都整体回滚,
|
||
// 不会出现"作业已交给 release 但阶段还停在 running"或"阶段成功但作业没交接"。
|
||
func (s *Store) CompleteLibraryHandoff(ctx context.Context, libraryJobID, ownerQueueID string,
|
||
request jobs.EnqueueRequest, detail string, now time.Time) (bool, error) {
|
||
tx, err := s.db.Begin(ctx)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
if _, owned, err := s.lockLibraryJob(ctx, tx, libraryJobID, ownerQueueID); err != nil {
|
||
return false, err
|
||
} else if !owned {
|
||
return false, nil
|
||
}
|
||
if _, err := scanJob(tx.QueryRow(ctx, enqueueJobSQL, request.ID, request.Type, request.Payload,
|
||
request.Priority, request.AvailableAt, request.MaxAttempts, request.IdempotencyKey, now)); err != nil {
|
||
return false, fmt.Errorf("enqueue release job: %w", err)
|
||
}
|
||
tag, err := tx.Exec(ctx, `UPDATE library_jobs SET queue_job_id=$2, updated_at=$3
|
||
WHERE id=$1 AND queue_job_id=$4`, libraryJobID, request.ID, now.UTC(), ownerQueueID)
|
||
if err != nil {
|
||
return false, fmt.Errorf("hand over library job: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return false, errors.New("hand over library job: 所有权在事务内已变化,回滚")
|
||
}
|
||
if err := upsertLibraryStage(ctx, tx, libraryJobID, "handoff", "succeeded", detail, now); err != nil {
|
||
return false, err
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return false, err
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
// FailLibraryHandoff 同一个事务里写"阶段失败 + 作业失败",不留半条记录。
|
||
func (s *Store) FailLibraryHandoff(ctx context.Context, libraryJobID, ownerQueueID, message string,
|
||
now time.Time) (bool, error) {
|
||
tx, err := s.db.Begin(ctx)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
if _, owned, err := s.lockLibraryJob(ctx, tx, libraryJobID, ownerQueueID); err != nil {
|
||
return false, err
|
||
} else if !owned {
|
||
return false, nil
|
||
}
|
||
if err := upsertLibraryStage(ctx, tx, libraryJobID, "handoff", "failed", message, now); err != nil {
|
||
return false, err
|
||
}
|
||
tag, err := tx.Exec(ctx, `
|
||
UPDATE library_jobs SET state='failed', stage='handoff', error=$3, recovery=$4,
|
||
updated_at=$2, finished_at=$2
|
||
WHERE id=$1 AND queue_job_id=$5 AND state IN ('pending','running')`,
|
||
libraryJobID, now.UTC(), message, "未进入维护窗口、未写任何知识;修正后重试", ownerQueueID)
|
||
if err != nil {
|
||
return false, fmt.Errorf("fail library handoff: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return false, errors.New("fail library handoff: 所有权在事务内已变化,回滚")
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return false, err
|
||
}
|
||
return true, nil
|
||
}
|