98294b9583
build / build (push) Failing after 24s
why:审计 19 项(12 P1 / 7 P2)来自同一批根因,逐点试错会把协议撕成十几份:
· 意图与执行混为一谈(F01/F15):`library.sync:<scope>:<快照>` 是终身幂等键,
A→B→A 的第三次发布拿到第一次的历史成功作业,什么都不做;
· 版本行既是实体又是事件(F02/F09):主键只由 (kind,scope,快照) 决定、写入是
upsert,第二个作业覆盖 `source_job_id`/`evidence`/`created_at`;
· 接管没有代次(F13):两个 worker 都能认领同一 pending 作业,失租的旧执行还能
写业务状态、释放别人的维护锁;
· 早退与终态没有统一政策(F03/F04/F05/F06):半写窗口的前置检查失败会重新开门、
服务起不来也照样开入口、journal 终态后收尾不可重建、恢复成功不回写 current;
· 数据身份与准入各写一套(F07/F08/F14/F17):接管丢了复用语义、全库搬运不改
location_hash、通用入口可造 library.* 作业、目标模型从没被问过;
· 存储/容量/UI/CI 判据各自为政(F10/F11/F12/F16/F18/F19)。
what:按批准的一套协议集中实施(详见 handoffs/release-consolidated-implementation/
current-contract.md):
· operation(一次明确意图)/ attempt(epoch 代次)/ 不可变版本事件 / current 指针;
治理库迁移 76,只动治理库,不碰 WeKnora Schema/源码/镜像;
· 所有业务状态、阶段、版本事件、维护锁写入带 epoch 守卫(失租代次写不进去);
心跳失租立刻取消 handler;
· 半写窗口早退保持关闭;服务未全部健康绝不开入口;journal 终态可从持久事实重建
收尾;恢复成功登记 recovery 事件并推进 current;
· worker 在动维护前核对业务行/kind/scope/队列关联;通用入口拒绝 library.*;
· 全库搬运重算 location_hash 并与文档级导入共用实现;接管消费写集合的复用决定;
发布前核目标实时模型契约、容量门禁前置;
· 存储保护闭包改为全量依赖、类型只认显式登记、保护未知则不出可删候选;
· 页面核 HTTP 状态与结构、恢复纳入轮询、Wiki 差异覆盖标题/摘要/父级。
how:`make verify` rc=0(16 项必需检查全绿,含 go test 全量与前端 485 例);
`go vet ./...` rc=0;`npx tsc -p tsconfig.json --noEmit` rc=0;逐项负例见各包测试
(F03/F04/F08/F12/F13/F14/F16/F18/F19 都有先红后绿记录)。
未执行:治理库迁移到任何已有库、真实 PG/ES/S3/WeKnora 集成、故障注入、部署。
692 lines
32 KiB
Go
692 lines
32 KiB
Go
package postgres
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
"github.com/jackc/pgx/v5/pgconn"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// 整库作业与版本:队列(jobs)负责租约重试,本文件负责业务身份、阶段与结果。
|
||
// 互斥与幂等都由数据库唯一索引保证,不依赖前端或单 worker 部署。
|
||
|
||
func (s *Store) CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error) {
|
||
request := job.RequestDigest
|
||
if request == "" {
|
||
return governance.LibraryJob{}, false, errors.New("library job requires a request digest")
|
||
}
|
||
rawRequest, err := json.Marshal(map[string]any{"digest": request, "kind": string(job.Kind), "scope": job.Scope})
|
||
if err != nil {
|
||
return governance.LibraryJob{}, false, err
|
||
}
|
||
var id string
|
||
err = s.db.QueryRow(ctx, `
|
||
INSERT INTO library_jobs (id, scope, kind, state, stage, idempotency_key, request_digest, request,
|
||
queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, operation_id, created_at, updated_at)
|
||
VALUES ($1,$2,$3,'pending','',$4,$5,$6,$7,$8,$9,$10,'','',$11,$12,$12)
|
||
ON CONFLICT (idempotency_key) DO NOTHING
|
||
RETURNING id`,
|
||
job.ID, job.Scope, string(job.Kind), job.IdempotencyKey, request, rawRequest,
|
||
job.QueueJobID, job.SnapshotID, job.TargetSnapshotID, job.Actor, job.OperationID, now.UTC()).Scan(&id)
|
||
if err != nil {
|
||
var pgErr *pgconn.PgError
|
||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||
// 同一目标的写作业(sync/restore)共用一个唯一索引,跨 kind 互斥。
|
||
if pgErr.ConstraintName == "library_jobs_write_target" || pgErr.ConstraintName == "library_jobs_active" {
|
||
return governance.LibraryJob{}, false, governance.ErrLibraryJobActive
|
||
}
|
||
}
|
||
if !errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryJob{}, false, fmt.Errorf("create library job: %w", err)
|
||
}
|
||
// 幂等键已存在:必须返回**已存**作业,并核对请求体是否一致。
|
||
existing, loadErr := s.libraryJobByKey(ctx, job.IdempotencyKey)
|
||
if loadErr != nil {
|
||
return governance.LibraryJob{}, false, loadErr
|
||
}
|
||
if existing.RequestDigest != request {
|
||
return existing, false, governance.ErrLibraryJobIdempotencyConflict
|
||
}
|
||
return existing, true, nil
|
||
}
|
||
for _, stage := range stages {
|
||
if _, err := s.db.Exec(ctx, `
|
||
INSERT INTO library_job_stages (job_id, stage, state, detail, updated_at)
|
||
VALUES ($1,$2,'pending','',$3)
|
||
ON CONFLICT (job_id, stage) DO NOTHING`, id, stage, now.UTC()); err != nil {
|
||
return governance.LibraryJob{}, false, fmt.Errorf("create library job stage: %w", err)
|
||
}
|
||
}
|
||
created, err := s.LibraryJob(ctx, id)
|
||
return created, false, err
|
||
}
|
||
|
||
func (s *Store) libraryJobByKey(ctx context.Context, key string) (governance.LibraryJob, error) {
|
||
var id string
|
||
if err := s.db.QueryRow(ctx, `SELECT id FROM library_jobs WHERE idempotency_key=$1`, key).Scan(&id); err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryJob{}, errors.New("library job idempotency key not found")
|
||
}
|
||
return governance.LibraryJob{}, fmt.Errorf("load library job by key: %w", err)
|
||
}
|
||
return s.LibraryJob(ctx, id)
|
||
}
|
||
|
||
func (s *Store) LibraryJob(ctx context.Context, id string) (governance.LibraryJob, error) {
|
||
var job governance.LibraryJob
|
||
var kind, state string
|
||
var finished *time.Time
|
||
if err := s.db.QueryRow(ctx, `
|
||
SELECT id, scope, kind, state, stage, idempotency_key, request_digest, queue_job_id, snapshot_id,
|
||
target_snapshot_id, actor, error, recovery, operation_id, attempt_epoch, created_at, updated_at, finished_at
|
||
FROM library_jobs WHERE id=$1`, id).Scan(&job.ID, &job.Scope, &kind, &state, &job.Stage, &job.IdempotencyKey,
|
||
&job.RequestDigest, &job.QueueJobID, &job.SnapshotID, &job.TargetSnapshotID, &job.Actor, &job.Error,
|
||
&job.Recovery, &job.OperationID, &job.AttemptEpoch, &job.CreatedAt, &job.UpdatedAt, &finished); err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryJob{}, err
|
||
}
|
||
return governance.LibraryJob{}, fmt.Errorf("load library job: %w", err)
|
||
}
|
||
job.Kind, job.State, job.FinishedAt = governance.LibraryJobKind(kind), governance.LibraryJobState(state), finished
|
||
stages, err := s.libraryStages(ctx, id)
|
||
if err != nil {
|
||
return governance.LibraryJob{}, err
|
||
}
|
||
job.Stages = stages
|
||
return job, nil
|
||
}
|
||
|
||
// LibraryReuseSource 是一条"这个目标的持久来源"记录:成功同步过的快照,
|
||
// 或这个目标自己的预备份。资源复用只认它作来源证据。
|
||
type LibraryReuseSource struct {
|
||
JobID string
|
||
SnapshotID string
|
||
Kind string // sync | prebackup
|
||
FinishedAt time.Time
|
||
}
|
||
|
||
// LibraryReuseSources 读这个目标的持久来源(含已被恢复顶替的生效版本)。
|
||
// 同步来源要求同时存在 production_effective 版本登记:只写成功但版本没登记的
|
||
// 作业不算可审计来源。
|
||
func (s *Store) LibraryReuseSources(ctx context.Context, scope string, limit int) ([]LibraryReuseSource, error) {
|
||
if limit < 1 || limit > 200 {
|
||
limit = 100
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT j.id, j.target_snapshot_id, 'sync', coalesce(j.finished_at, j.updated_at)
|
||
FROM library_jobs j
|
||
WHERE j.scope=$1 AND j.kind='sync' AND j.state='succeeded' AND coalesce(j.target_snapshot_id,'')<>''
|
||
AND EXISTS (SELECT 1 FROM library_versions v
|
||
WHERE v.scope=j.scope AND v.kind='production_effective'
|
||
AND v.snapshot_id=j.target_snapshot_id AND v.source_job_id=j.id)
|
||
UNION ALL
|
||
SELECT coalesce(v.source_job_id,''), v.snapshot_id, 'prebackup', v.created_at
|
||
FROM library_versions v
|
||
WHERE v.scope=$1 AND v.kind='production_prebackup' AND coalesce(v.snapshot_id,'')<>''
|
||
ORDER BY 4 DESC
|
||
LIMIT $2`, scope, limit)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("load library reuse sources: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
sources := make([]LibraryReuseSource, 0, 8)
|
||
for rows.Next() {
|
||
var source LibraryReuseSource
|
||
if err := rows.Scan(&source.JobID, &source.SnapshotID, &source.Kind, &source.FinishedAt); err != nil {
|
||
return nil, fmt.Errorf("scan library reuse source: %w", err)
|
||
}
|
||
sources = append(sources, source)
|
||
}
|
||
return sources, rows.Err()
|
||
}
|
||
|
||
func (s *Store) libraryStages(ctx context.Context, id string) ([]governance.LibraryStage, error) {
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT stage, state, total, completed, bytes_total, bytes_done, detail, updated_at
|
||
FROM library_job_stages WHERE job_id=$1 ORDER BY updated_at, stage`, id)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("load library job stages: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
result := make([]governance.LibraryStage, 0, 7)
|
||
for rows.Next() {
|
||
var stage governance.LibraryStage
|
||
if err := rows.Scan(&stage.Stage, &stage.State, &stage.Total, &stage.Completed, &stage.BytesTotal,
|
||
&stage.BytesDone, &stage.Detail, &stage.UpdatedAt); err != nil {
|
||
return nil, fmt.Errorf("scan library job stage: %w", err)
|
||
}
|
||
result = append(result, stage)
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
// ListLibraryJobs 返回某目标最近的作业(按创建时间倒序),供页面在重启后恢复展示。
|
||
func (s *Store) ListLibraryJobs(ctx context.Context, scope string, limit int) ([]governance.LibraryJob, error) {
|
||
if limit < 1 || limit > 50 {
|
||
return nil, errors.New("library job list limit out of range")
|
||
}
|
||
rows, err := s.db.Query(ctx, `SELECT id FROM library_jobs WHERE scope=$1 ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list library jobs: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
ids := make([]string, 0, limit)
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil, fmt.Errorf("scan library job id: %w", err)
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
result := make([]governance.LibraryJob, 0, len(ids))
|
||
for _, id := range ids {
|
||
job, err := s.LibraryJob(ctx, id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result = append(result, job)
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// UpdateLibraryStage 只改调用方给了值的维度:nil 表示"这一维还没有可靠数据",
|
||
// 保持 NULL,绝不写成 0。
|
||
func (s *Store) UpdateLibraryStage(ctx context.Context, jobID, stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error {
|
||
tag, err := s.db.Exec(ctx, `
|
||
UPDATE library_job_stages SET
|
||
state = CASE WHEN $3='' THEN state ELSE $3 END,
|
||
total = COALESCE($4, total),
|
||
completed = COALESCE($5, completed),
|
||
bytes_total = COALESCE($6, bytes_total),
|
||
bytes_done = COALESCE($7, bytes_done),
|
||
detail = CASE WHEN $8='' THEN detail ELSE $8 END,
|
||
updated_at = $9
|
||
WHERE job_id=$1 AND stage=$2`, jobID, stage, state, total, completed, bytesTotal, bytesDone, detail, now.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("update library stage: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return errors.New("library stage not found")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// AttachLibraryQueueJob 把队列作业号写到已建的业务作业上(两步创建,避免先入队后才发现互斥冲突)。
|
||
func (s *Store) AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error {
|
||
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET queue_job_id=$2, updated_at=$3 WHERE id=$1`, jobID, queueJobID, now.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("attach library queue job: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return errors.New("library job not found")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *Store) CancelLibraryJob(ctx context.Context, jobID, reason string, now time.Time) error {
|
||
return s.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobCancelled, "", reason, "", "", "", now)
|
||
}
|
||
|
||
func (s *Store) AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error {
|
||
finished := any(nil)
|
||
if state == governance.LibraryJobSucceeded || state == governance.LibraryJobFailed || state == governance.LibraryJobCancelled {
|
||
finished = now.UTC()
|
||
}
|
||
tag, err := s.db.Exec(ctx, `
|
||
UPDATE library_jobs SET
|
||
state=$2,
|
||
stage = CASE WHEN $3='' THEN stage ELSE $3 END,
|
||
error = CASE WHEN $4='' THEN error ELSE $4 END,
|
||
recovery = CASE WHEN $5='' THEN recovery ELSE $5 END,
|
||
snapshot_id = CASE WHEN $6='' THEN snapshot_id ELSE $6 END,
|
||
target_snapshot_id = CASE WHEN $7='' THEN target_snapshot_id ELSE $7 END,
|
||
updated_at=$8,
|
||
finished_at=COALESCE($9, finished_at)
|
||
WHERE id=$1`, jobID, string(state), stage, message, recovery, snapshotID, targetSnapshotID, now.UTC(), finished)
|
||
if err != nil {
|
||
return fmt.Errorf("advance library job: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return errors.New("library job not found")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// RecordLibraryVersion 写入一条**不可变版本事件**,并在需要时推进 current 指针。
|
||
//
|
||
// 与旧实现的区别(F02/F09 的根因):
|
||
// - 事件 ID 含 operation:不同 operation 引用同一快照是两条事件,`source_job_id`/
|
||
// `evidence`/`created_at` 谁也不覆盖谁(旧实现是 `ON CONFLICT DO UPDATE` 整份替换)。
|
||
// - 同一 operation 重放是幂等:`ON CONFLICT (id) DO NOTHING`,已存行原样返回。
|
||
// - "谁当前生效"由 `library_current_versions` 表达,用单调次序做条件更新;
|
||
// 更早登记的事实不能把指针拉回历史,旧 epoch 在写边界就被守卫拒绝(见 guard)。
|
||
// - 生效/恢复事件才动指针;预备份/封存只追加事件。
|
||
func (s *Store) RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error {
|
||
evidence, err := json.Marshal(version.Evidence)
|
||
if err != nil {
|
||
return fmt.Errorf("encode library version evidence: %w", err)
|
||
}
|
||
tx, err := s.db.Begin(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("begin library version: %w", err)
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
|
||
// 执行代次守卫:写事件的人必须还是当前代次(失租旧执行器不能登记"我成功了")。
|
||
if version.SourceJobID != "" && version.AttemptEpoch > 0 {
|
||
if err := guardLibraryJobEpoch(ctx, tx, version.SourceJobID, version.AttemptEpoch, version.AttemptOwner); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
|
||
tag, err := tx.Exec(ctx, `
|
||
INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id,
|
||
operation_id, attempt_epoch, evidence, created_by, created_at)
|
||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
|
||
ON CONFLICT (id) DO NOTHING`,
|
||
version.ID, version.Scope, string(version.Kind), version.SnapshotID, version.VersionLabel,
|
||
version.SourceJobID, version.OperationID, version.AttemptEpoch, evidence, version.CreatedBy,
|
||
version.CreatedAt.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("record library version event: %w", err)
|
||
}
|
||
inserted := tag.RowsAffected() == 1
|
||
|
||
if version.Kind == governance.LibraryVersionProductionEffective || version.Kind == governance.LibraryVersionRecoveryVerified {
|
||
if err := advanceLibraryCurrent(ctx, tx, version, inserted); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return tx.Commit(ctx)
|
||
}
|
||
|
||
// advanceLibraryCurrent 用单调次序条件推进 current 指针。
|
||
//
|
||
// 次序键 = (事件登记时间|operation|epoch)。它只回答"哪一条事实更晚被登记",
|
||
// 不回答"哪个意图更重要";旧代次能不能写由 epoch 守卫回答,两件事分开才不会互相冒充。
|
||
func advanceLibraryCurrent(ctx context.Context, tx pgx.Tx, version governance.LibraryVersion, inserted bool) error {
|
||
orderKey := governance.CurrentOrderKey(version.CreatedAt, version.OperationID, version.AttemptEpoch)
|
||
if !inserted {
|
||
// 重放:事件已经在台账里。确认指针没有被别的更晚事实覆盖,否则只确认自己的历史。
|
||
var currentID string
|
||
err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1 FOR UPDATE`,
|
||
version.Scope).Scan(¤tID)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return nil
|
||
}
|
||
if err != nil {
|
||
return fmt.Errorf("load library current pointer: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
tag, err := tx.Exec(ctx, `
|
||
INSERT INTO library_current_versions (scope, version_id, snapshot_id, operation_id, attempt_epoch, order_key, updated_at)
|
||
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
||
ON CONFLICT (scope) DO UPDATE SET
|
||
version_id=EXCLUDED.version_id, snapshot_id=EXCLUDED.snapshot_id,
|
||
operation_id=EXCLUDED.operation_id, attempt_epoch=EXCLUDED.attempt_epoch,
|
||
order_key=EXCLUDED.order_key, updated_at=EXCLUDED.updated_at
|
||
WHERE EXCLUDED.order_key > library_current_versions.order_key`,
|
||
version.Scope, version.ID, version.SnapshotID, version.OperationID, version.AttemptEpoch, orderKey,
|
||
version.CreatedAt.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("advance library current pointer: %w", err)
|
||
}
|
||
var currentID string
|
||
if err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1`,
|
||
version.Scope).Scan(¤tID); err != nil {
|
||
return fmt.Errorf("read back library current pointer: %w", err)
|
||
}
|
||
if tag.RowsAffected() == 0 && currentID != version.ID {
|
||
return fmt.Errorf("%w(scope=%s:当前 %s,本条 %s)", governance.ErrLibraryCurrentStale,
|
||
version.Scope, currentID, version.ID)
|
||
}
|
||
// superseded_at 只是展示元数据("这条已被顶替"),不改任何证据字段。
|
||
// 它跟着指针走:指针指向谁,谁就是未顶替的那一行。
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE library_versions SET superseded_at = CASE WHEN id=$2 THEN NULL ELSE COALESCE(superseded_at, $3) END
|
||
WHERE scope=$1 AND kind IN ('production_effective','recovery_verified')`, version.Scope, currentID,
|
||
version.CreatedAt.UTC()); err != nil {
|
||
return fmt.Errorf("refresh library version supersede metadata: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// InterruptedLibraryJobs 找出"业务作业 running、但队列里已没有活跃作业"的作业:
|
||
// worker 被杀/重启后它们需要被接管,而不是被当成失败丢掉已完成的预备份。
|
||
func (s *Store) InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error) {
|
||
// lease 过期才算中断:worker 刚被杀但 lease 未到期时,队列里还是 running,
|
||
// 直接判"没中断"会让作业永远悬着;所以这里按 lease_until 判断。
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT lj.id FROM library_jobs lj
|
||
LEFT JOIN jobs j ON j.id=lj.queue_job_id
|
||
WHERE lj.kind IN ('sync','restore','baseline')
|
||
AND (
|
||
-- 业务作业在跑、队列已经没了/停了/租约过期
|
||
(lj.state='running' AND (j.id IS NULL OR j.state IN ('failed','completed','cancelled','dead')
|
||
OR (j.state IN ('pending','running','retry') AND j.lease_until IS NOT NULL AND j.lease_until < now())))
|
||
-- 业务作业已被前一轮接管置为 pending,但队列作业的租约也没了(worker 又死了)
|
||
OR (lj.state='pending' AND j.state='running' AND j.lease_until IS NOT NULL AND j.lease_until < now())
|
||
)
|
||
ORDER BY lj.created_at`)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list interrupted library jobs: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
ids := make([]string, 0, 4)
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil, err
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
jobs := make([]governance.LibraryJob, 0, len(ids))
|
||
for _, id := range ids {
|
||
job, err := s.LibraryJob(ctx, id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
jobs = append(jobs, job)
|
||
}
|
||
return jobs, nil
|
||
}
|
||
|
||
// ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零,
|
||
// 否则 attempt<max_attempts 永远不成立,作业再也不会被任何 worker 领取。
|
||
//
|
||
// 但**不能清掉仍然有效的租约**:那等于把正在执行的那一代直接踢掉,而它可能已经在写目标
|
||
// (F13 的原始形态)。只有租约为空或已过期才允许重置。
|
||
func (s *Store) ResetQueueJobForResume(ctx context.Context, queueJobID string, now time.Time) error {
|
||
tag, err := s.db.Exec(ctx, `UPDATE jobs
|
||
SET state='pending', attempt=0, lease_owner=NULL, lease_until=NULL, available_at=$2, updated_at=$2
|
||
WHERE id=$1 AND state IN ('running','retry','failed','pending')
|
||
AND (lease_until IS NULL OR lease_until < $2)`, queueJobID, now.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("reset queue job for resume: %w", err)
|
||
}
|
||
if tag.RowsAffected() == 0 {
|
||
// 有活跃租约:另一个执行者正在跑,本次接管必须让路(调用方跳过即可)。
|
||
return ErrLibraryAttemptHeld
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ClaimLibraryJobForResume 用一次条件更新把中断作业抢过来:只有把 running 改成 pending
|
||
// 的那一个 worker 拿到 true,多个 worker 同时重启也只会有一个执行(fencing)。
|
||
//
|
||
// 条件里必须看租约:pending→pending 的无条件认领会让两个 worker 都"认领成功",
|
||
// 然后各自跑一遍同一个窗口(F13)。租约仍有效时直接失败,让调用方跳过。
|
||
func (s *Store) ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error) {
|
||
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET state='pending', updated_at=$2
|
||
WHERE id=$1 AND state IN ('running','pending')
|
||
AND (lease_owner='' OR lease_until IS NULL OR lease_until < $2)`, jobID, now.UTC())
|
||
if err != nil {
|
||
return false, fmt.Errorf("claim interrupted library job: %w", err)
|
||
}
|
||
return tag.RowsAffected() == 1, nil
|
||
}
|
||
|
||
// ReconcileLibraryJobs 收拾"业务作业还在飞、队列里已经没有对应作业"的孤儿:
|
||
// 它们会永久占用 (scope,kind) 唯一索引,让同一目标再也发不出作业。
|
||
// 只标失败并写明恢复责任,不猜结果、不重放。
|
||
func (s *Store) ReconcileLibraryJobs(ctx context.Context, now time.Time) (int, error) {
|
||
if _, err := s.db.Exec(ctx, `
|
||
UPDATE library_jobs AS lj
|
||
SET state='failed', error='队列中已无对应作业,业务作业未收到结果', recovery='封存作业由 agent 恢复来源;确认来源 healthy 后可重发', updated_at=$1, finished_at=$1
|
||
WHERE lj.state IN ('pending','running')
|
||
AND lj.queue_job_id <> ''
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM jobs AS j
|
||
WHERE j.id = lj.queue_job_id AND j.state IN ('pending','running','retry')
|
||
)`, now.UTC()); err != nil {
|
||
return 0, fmt.Errorf("reconcile library jobs: %w", err)
|
||
}
|
||
// 建了业务作业但队列投递没完成(queue_job_id 为空)超过 5 分钟,也是孤儿。
|
||
tag, err := s.db.Exec(ctx, `
|
||
UPDATE library_jobs
|
||
SET state='failed', error='队列投递未完成,作业从未开始', recovery='无需恢复来源;可直接重发', updated_at=$1, finished_at=$1
|
||
WHERE state IN ('pending','running') AND queue_job_id='' AND created_at < $2`, now.UTC(), now.Add(-5*time.Minute).UTC())
|
||
if err != nil {
|
||
return 0, fmt.Errorf("reconcile unattached library jobs: %w", err)
|
||
}
|
||
return int(tag.RowsAffected()), nil
|
||
}
|
||
|
||
// LibraryVersionByRef 按 (scope, kind, 作业, 快照) 精确定位一条版本记录。
|
||
//
|
||
// 为什么不能"取最近 N 条再在内存里筛":那是窗口,历史一旦超过窗口上限,
|
||
// 登记/接管/恢复就会把"存在"读成"不存在"——真实现场是写集合的引用无法登记,
|
||
// 报"读不到现有版本证据 / library version limit out of range"。
|
||
// 身份查询必须走索引条件,不许依赖排序窗口(把上限从 200 改成 100 只是换个数)。
|
||
//
|
||
// 同一 (scope, kind, snapshot) 只该有一行(id 由这三者决定,写入是 upsert)。
|
||
// 命中多行、或命中的那行归别的作业所有,都说明台账不唯一:返回错误让调用方
|
||
// 拒绝,不许挑一条"看起来对"的记录去恢复生产。
|
||
func (s *Store) LibraryVersionByRef(ctx context.Context, scope string, kind governance.LibraryVersionKind,
|
||
sourceJobID, snapshotID string) (governance.LibraryVersion, bool, error) {
|
||
if scope == "" || kind == "" || sourceJobID == "" || snapshotID == "" {
|
||
return governance.LibraryVersion{}, false, errors.New("library version ref requires scope, kind, job and snapshot")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at
|
||
FROM library_versions
|
||
WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4
|
||
ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), sourceJobID, snapshotID)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("load library version by ref: %w", err)
|
||
}
|
||
items, err := scanLibraryVersions(rows, 2)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, err
|
||
}
|
||
if len(items) > 1 {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("同一身份存在 %d 条版本记录(作业 %s,快照 %s):拒绝挑选",
|
||
len(items), sourceJobID, snapshotID)
|
||
}
|
||
if len(items) == 1 {
|
||
return items[0], true, nil
|
||
}
|
||
// 没命中时要区分"确实没有"与"这一行归别的作业":后者是身份冲突,
|
||
// 当成"没有记录"就会允许调用方覆盖别人的台账。
|
||
var owner string
|
||
err = s.db.QueryRow(ctx, `
|
||
SELECT source_job_id FROM library_versions WHERE scope=$1 AND kind=$2 AND snapshot_id=$3
|
||
ORDER BY created_at DESC LIMIT 1`, scope, string(kind), snapshotID).Scan(&owner)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryVersion{}, false, nil
|
||
}
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("load library version owner: %w", err)
|
||
}
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("版本记录(快照 %s)属于作业 %s,不是 %s:拒绝借用",
|
||
snapshotID, owner, sourceJobID)
|
||
}
|
||
|
||
// scanLibraryVersions 是版本行唯一的扫描实现:两个查询(窗口列表与按身份精确查)
|
||
// 必须对同一种形状解码,否则"查得到"和"读得对"会分叉。
|
||
func scanLibraryVersions(rows pgx.Rows, capacity int) ([]governance.LibraryVersion, error) {
|
||
defer rows.Close()
|
||
result := make([]governance.LibraryVersion, 0, capacity)
|
||
for rows.Next() {
|
||
var item governance.LibraryVersion
|
||
var kind string
|
||
var evidence []byte
|
||
if err := rows.Scan(&item.ID, &item.Scope, &kind, &item.SnapshotID, &item.VersionLabel, &item.SourceJobID,
|
||
&item.OperationID, &item.AttemptEpoch, &evidence, &item.CreatedBy, &item.CreatedAt, &item.SupersededAt); err != nil {
|
||
return nil, fmt.Errorf("scan library version: %w", err)
|
||
}
|
||
item.Kind = governance.LibraryVersionKind(kind)
|
||
if len(evidence) > 0 {
|
||
_ = json.Unmarshal(evidence, &item.Evidence)
|
||
}
|
||
result = append(result, item)
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
// LibraryVersions 是**分页展示**用的窗口列表(1..100),语义是"最近 N 条"。
|
||
// 任何"当前生效 / 本快照是否生效过"的判断都不许用它:窗口一满就会把存在读成不存在。
|
||
func (s *Store) LibraryVersions(ctx context.Context, scope string, limit int) ([]governance.LibraryVersion, error) {
|
||
if limit < 1 || limit > 100 {
|
||
return nil, errors.New("library version limit out of range")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at
|
||
FROM library_versions WHERE scope=$1 ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list library versions: %w", err)
|
||
}
|
||
return scanLibraryVersions(rows, limit)
|
||
}
|
||
|
||
// LibraryCurrentVersion 返回这个 scope **当前生效**的事实(由指针表回答)。
|
||
//
|
||
// 三个理由必须一起满足:
|
||
//
|
||
// · 不能用窗口列表——生效事实可能很久以前登记,之后写进来的预备份/封存会把它挤出
|
||
// 任何"最近 N 条",接管/基线/闸口就会把"已生效"读成"没有生效"。
|
||
// · 不能靠扫 library_versions.kind='production_effective' 挑未顶替行——事件不可变,
|
||
// "谁是当前"是指针的事实,不是事件行的可变字段(F02)。
|
||
// · 指针每 scope 至多一行(主键),所以这里不需要"挑一条看起来对的"。
|
||
//
|
||
// 指针可能指向 production_effective,也可能指向 recovery_verified(按预备份恢复后
|
||
// 的当前内容身份)——调用方只能用它回答"当前是什么内容",不能假设它是一次成功发布。
|
||
func (s *Store) LibraryCurrentVersion(ctx context.Context, scope string) (governance.LibraryVersion, bool, error) {
|
||
if scope == "" {
|
||
return governance.LibraryVersion{}, false, errors.New("library current version requires scope")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT v.id, v.scope, v.kind, v.snapshot_id, v.version_label, v.source_job_id, v.operation_id, v.attempt_epoch,
|
||
v.evidence, v.created_by, v.created_at, v.superseded_at
|
||
FROM library_current_versions c JOIN library_versions v ON v.id = c.version_id
|
||
WHERE c.scope=$1`, scope)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("load current library version: %w", err)
|
||
}
|
||
items, err := scanLibraryVersions(rows, 2)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, err
|
||
}
|
||
if len(items) > 1 {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("scope=%s 的当前指针指向 %d 行事件:台账被写坏,拒绝挑选",
|
||
scope, len(items))
|
||
}
|
||
if len(items) == 1 {
|
||
return items[0], true, nil
|
||
}
|
||
return governance.LibraryVersion{}, false, nil
|
||
}
|
||
|
||
// LibraryVersionOfSnapshot 按 (scope, kind, 快照) 取这个快照**自己的**登记记录(不限作业)。
|
||
//
|
||
// 用途是"本快照曾经生效过吗/是否已被顶替":版本 id 由 (kind, scope, 快照) 决定,
|
||
// 一个快照在一种类型下只该有一行,作业归属会随恢复/重放被改写,所以不能用作业当条件。
|
||
// 命中多行同样拒绝挑选。
|
||
func (s *Store) LibraryVersionOfSnapshot(ctx context.Context, scope string,
|
||
kind governance.LibraryVersionKind, snapshotID string) (governance.LibraryVersion, bool, error) {
|
||
if scope == "" || kind == "" || snapshotID == "" {
|
||
return governance.LibraryVersion{}, false, errors.New("library version snapshot requires scope, kind and snapshot")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at
|
||
FROM library_versions
|
||
WHERE scope=$1 AND kind=$2 AND snapshot_id=$3
|
||
ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), snapshotID)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, fmt.Errorf("load library version of snapshot: %w", err)
|
||
}
|
||
found, err := scanLibraryVersions(rows, 2)
|
||
if err != nil {
|
||
return governance.LibraryVersion{}, false, err
|
||
}
|
||
// 同一快照被多次意图引用时会有多条事件(每次 operation 一条),这是**正常历史**,
|
||
// 不是台账损坏:按登记时间取最新一条回答"这个快照最近一次登记成什么"。
|
||
// 谁当前生效不在这里判断,那是 current 指针的事实。
|
||
if len(found) >= 1 {
|
||
return found[0], true, nil
|
||
}
|
||
return governance.LibraryVersion{}, false, nil
|
||
}
|
||
|
||
// CreateLibraryAuthorization 记录一次生产发布授权;同一 (scope,snapshot,digest) 只保留最新有效。
|
||
func (s *Store) CreateLibraryAuthorization(ctx context.Context, item governance.LibraryReleaseAuthorization) error {
|
||
if item.ID == "" || item.Scope == "" || item.SnapshotID == "" || item.ParamsDigest == "" || item.ExpiresAt.IsZero() {
|
||
return errors.New("invalid library release authorization")
|
||
}
|
||
_, err := s.db.Exec(ctx, `
|
||
INSERT INTO library_release_authorizations (id, scope, snapshot_id, params_digest, authorized_by, expires_at, created_at)
|
||
VALUES ($1,$2,$3,$4,$5,$6,$7)`,
|
||
item.ID, item.Scope, item.SnapshotID, item.ParamsDigest, item.AuthorizedBy, item.ExpiresAt.UTC(), item.CreatedAt.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("create library release authorization: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ActiveLibraryAuthorization 只返回"未撤销、未过期、且参数摘要完全一致"的授权。
|
||
// 参数一变就查不到,自然拒绝,而不是靠调用方记得复核。
|
||
func (s *Store) ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, paramsDigest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error) {
|
||
var item governance.LibraryReleaseAuthorization
|
||
err := s.db.QueryRow(ctx, `
|
||
SELECT id, scope, snapshot_id, params_digest, authorized_by, expires_at, revoked_at, created_at
|
||
FROM library_release_authorizations
|
||
WHERE scope=$1 AND snapshot_id=$2 AND params_digest=$3 AND revoked_at IS NULL AND expires_at > $4
|
||
ORDER BY expires_at DESC, created_at DESC
|
||
LIMIT 1`, scope, snapshotID, paramsDigest, now.UTC()).Scan(&item.ID, &item.Scope, &item.SnapshotID,
|
||
&item.ParamsDigest, &item.AuthorizedBy, &item.ExpiresAt, &item.RevokedAt, &item.CreatedAt)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryReleaseAuthorization{}, false, nil
|
||
}
|
||
if err != nil {
|
||
return governance.LibraryReleaseAuthorization{}, false, fmt.Errorf("load library authorization: %w", err)
|
||
}
|
||
return item, true, nil
|
||
}
|
||
|
||
func (s *Store) RevokeLibraryAuthorization(ctx context.Context, id string, now time.Time) error {
|
||
tag, err := s.db.Exec(ctx, `UPDATE library_release_authorizations SET revoked_at=$2 WHERE id=$1 AND revoked_at IS NULL`, id, now.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("revoke library authorization: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return errors.New("library authorization not found or already revoked")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *Store) LibraryAuthorizations(ctx context.Context, scope string, limit int) ([]governance.LibraryReleaseAuthorization, error) {
|
||
if limit < 1 || limit > 100 {
|
||
return nil, errors.New("library authorization limit out of range")
|
||
}
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT id, scope, snapshot_id, params_digest, authorized_by, expires_at, revoked_at, created_at
|
||
FROM library_release_authorizations WHERE scope=$1 ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list library authorizations: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
result := make([]governance.LibraryReleaseAuthorization, 0, limit)
|
||
for rows.Next() {
|
||
var item governance.LibraryReleaseAuthorization
|
||
if err := rows.Scan(&item.ID, &item.Scope, &item.SnapshotID, &item.ParamsDigest, &item.AuthorizedBy,
|
||
&item.ExpiresAt, &item.RevokedAt, &item.CreatedAt); err != nil {
|
||
return nil, fmt.Errorf("scan library authorization: %w", err)
|
||
}
|
||
result = append(result, item)
|
||
}
|
||
return result, rows.Err()
|
||
}
|