- 新增 library_versions:测试封存、生产生效、生产预备份三类分开登记,
生产生效版本唯一(旧版本必须先 supersede),测试快照不能顶替生产版本
- 新增 library_jobs + library_job_stages:业务作业身份、阶段、错误与恢复说明,
分项计数用 NULL 表示"总量未知";互斥与幂等由数据库唯一索引保证
- 新增 GET /api/v1/library/overview、/jobs、/jobs/{id};POST /syncs 闸口未齐
返回 503 缺项且不建作业
- 封存作业落库:先建业务作业再入队,worker 推进 export 阶段并登记 test_sealed
- 发布管理读真实状态:生产未知不再顶替、差异 unknown、启动同步按缺项禁用、
同步明细/四类分项来自持久阶段;阶段勾选只认当前作业证据
- 准备信息标注封存时间与范围,不声称实时测试库已就绪
This commit is contained in:
@@ -0,0 +1,276 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 整库作业与版本:队列(jobs)负责租约重试,本文件负责业务身份、阶段与结果。
|
||||
// 互斥与幂等都由数据库唯一索引保证,不依赖前端或单 worker 部署。
|
||||
|
||||
func (s *Store) CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error) {
|
||||
request := job.RequestDigest
|
||||
if request == "" {
|
||||
return governance.LibraryJob{}, false, errors.New("library job requires a request digest")
|
||||
}
|
||||
rawRequest, err := json.Marshal(map[string]any{"digest": request, "kind": string(job.Kind), "scope": job.Scope})
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, false, err
|
||||
}
|
||||
var id string
|
||||
err = s.db.QueryRow(ctx, `
|
||||
INSERT INTO library_jobs (id, scope, kind, state, stage, idempotency_key, request_digest, request,
|
||||
queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, created_at, updated_at)
|
||||
VALUES ($1,$2,$3,'pending','',$4,$5,$6,$7,$8,$9,$10,'','',$11,$11)
|
||||
ON CONFLICT (idempotency_key) DO NOTHING
|
||||
RETURNING id`,
|
||||
job.ID, job.Scope, string(job.Kind), job.IdempotencyKey, request, rawRequest,
|
||||
job.QueueJobID, job.SnapshotID, job.TargetSnapshotID, job.Actor, now.UTC()).Scan(&id)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
if pgErr.ConstraintName == "library_jobs_active" {
|
||||
return governance.LibraryJob{}, false, governance.ErrLibraryJobActive
|
||||
}
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("create library job: %w", err)
|
||||
}
|
||||
// 幂等键已存在:必须返回**已存**作业,并核对请求体是否一致。
|
||||
existing, loadErr := s.libraryJobByKey(ctx, job.IdempotencyKey)
|
||||
if loadErr != nil {
|
||||
return governance.LibraryJob{}, false, loadErr
|
||||
}
|
||||
if existing.RequestDigest != request {
|
||||
return existing, false, governance.ErrLibraryJobIdempotencyConflict
|
||||
}
|
||||
return existing, true, nil
|
||||
}
|
||||
for _, stage := range stages {
|
||||
if _, err := s.db.Exec(ctx, `
|
||||
INSERT INTO library_job_stages (job_id, stage, state, detail, updated_at)
|
||||
VALUES ($1,$2,'pending','',$3)
|
||||
ON CONFLICT (job_id, stage) DO NOTHING`, id, stage, now.UTC()); err != nil {
|
||||
return governance.LibraryJob{}, false, fmt.Errorf("create library job stage: %w", err)
|
||||
}
|
||||
}
|
||||
created, err := s.LibraryJob(ctx, id)
|
||||
return created, false, err
|
||||
}
|
||||
|
||||
func (s *Store) libraryJobByKey(ctx context.Context, key string) (governance.LibraryJob, error) {
|
||||
var id string
|
||||
if err := s.db.QueryRow(ctx, `SELECT id FROM library_jobs WHERE idempotency_key=$1`, key).Scan(&id); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.LibraryJob{}, errors.New("library job idempotency key not found")
|
||||
}
|
||||
return governance.LibraryJob{}, fmt.Errorf("load library job by key: %w", err)
|
||||
}
|
||||
return s.LibraryJob(ctx, id)
|
||||
}
|
||||
|
||||
func (s *Store) LibraryJob(ctx context.Context, id string) (governance.LibraryJob, error) {
|
||||
var job governance.LibraryJob
|
||||
var kind, state string
|
||||
var finished *time.Time
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
SELECT id, scope, kind, state, stage, idempotency_key, request_digest, queue_job_id, snapshot_id,
|
||||
target_snapshot_id, actor, error, recovery, created_at, updated_at, finished_at
|
||||
FROM library_jobs WHERE id=$1`, id).Scan(&job.ID, &job.Scope, &kind, &state, &job.Stage, &job.IdempotencyKey,
|
||||
&job.RequestDigest, &job.QueueJobID, &job.SnapshotID, &job.TargetSnapshotID, &job.Actor, &job.Error,
|
||||
&job.Recovery, &job.CreatedAt, &job.UpdatedAt, &finished); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.LibraryJob{}, err
|
||||
}
|
||||
return governance.LibraryJob{}, fmt.Errorf("load library job: %w", err)
|
||||
}
|
||||
job.Kind, job.State, job.FinishedAt = governance.LibraryJobKind(kind), governance.LibraryJobState(state), finished
|
||||
stages, err := s.libraryStages(ctx, id)
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, err
|
||||
}
|
||||
job.Stages = stages
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (s *Store) libraryStages(ctx context.Context, id string) ([]governance.LibraryStage, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT stage, state, total, completed, bytes_total, bytes_done, detail, updated_at
|
||||
FROM library_job_stages WHERE job_id=$1 ORDER BY updated_at, stage`, id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load library job stages: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make([]governance.LibraryStage, 0, 7)
|
||||
for rows.Next() {
|
||||
var stage governance.LibraryStage
|
||||
if err := rows.Scan(&stage.Stage, &stage.State, &stage.Total, &stage.Completed, &stage.BytesTotal,
|
||||
&stage.BytesDone, &stage.Detail, &stage.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan library job stage: %w", err)
|
||||
}
|
||||
result = append(result, stage)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
// ListLibraryJobs 返回某目标最近的作业(按创建时间倒序),供页面在重启后恢复展示。
|
||||
func (s *Store) ListLibraryJobs(ctx context.Context, scope string, limit int) ([]governance.LibraryJob, error) {
|
||||
if limit < 1 || limit > 50 {
|
||||
return nil, errors.New("library job list limit out of range")
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `SELECT id FROM library_jobs WHERE scope=$1 ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list library jobs: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
ids := make([]string, 0, limit)
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("scan library job id: %w", err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]governance.LibraryJob, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
job, err := s.LibraryJob(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, job)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// UpdateLibraryStage 只改调用方给了值的维度:nil 表示"这一维还没有可靠数据",
|
||||
// 保持 NULL,绝不写成 0。
|
||||
func (s *Store) UpdateLibraryStage(ctx context.Context, jobID, stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_job_stages SET
|
||||
state = CASE WHEN $3='' THEN state ELSE $3 END,
|
||||
total = COALESCE($4, total),
|
||||
completed = COALESCE($5, completed),
|
||||
bytes_total = COALESCE($6, bytes_total),
|
||||
bytes_done = COALESCE($7, bytes_done),
|
||||
detail = CASE WHEN $8='' THEN detail ELSE $8 END,
|
||||
updated_at = $9
|
||||
WHERE job_id=$1 AND stage=$2`, jobID, stage, state, total, completed, bytesTotal, bytesDone, detail, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("update library stage: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return errors.New("library stage not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AttachLibraryQueueJob 把队列作业号写到已建的业务作业上(两步创建,避免先入队后才发现互斥冲突)。
|
||||
func (s *Store) AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET queue_job_id=$2, updated_at=$3 WHERE id=$1`, jobID, queueJobID, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("attach library queue job: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return errors.New("library job not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) CancelLibraryJob(ctx context.Context, jobID, reason string, now time.Time) error {
|
||||
return s.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobCancelled, "", reason, "", "", "", now)
|
||||
}
|
||||
|
||||
func (s *Store) AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error {
|
||||
finished := any(nil)
|
||||
if state == governance.LibraryJobSucceeded || state == governance.LibraryJobFailed || state == governance.LibraryJobCancelled {
|
||||
finished = now.UTC()
|
||||
}
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_jobs SET
|
||||
state=$2,
|
||||
stage = CASE WHEN $3='' THEN stage ELSE $3 END,
|
||||
error = CASE WHEN $4='' THEN error ELSE $4 END,
|
||||
recovery = CASE WHEN $5='' THEN recovery ELSE $5 END,
|
||||
snapshot_id = CASE WHEN $6='' THEN snapshot_id ELSE $6 END,
|
||||
target_snapshot_id = CASE WHEN $7='' THEN target_snapshot_id ELSE $7 END,
|
||||
updated_at=$8,
|
||||
finished_at=COALESCE($9, finished_at)
|
||||
WHERE id=$1`, jobID, string(state), stage, message, recovery, snapshotID, targetSnapshotID, now.UTC(), finished)
|
||||
if err != nil {
|
||||
return fmt.Errorf("advance library job: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return errors.New("library job not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecordLibraryVersion 写入一个版本记录。生产生效版本必须先把旧的 supersede 掉,
|
||||
// 否则唯一索引会拒绝——这正是"不许两个生效版本并存"的保证。
|
||||
func (s *Store) RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error {
|
||||
evidence, err := json.Marshal(version.Evidence)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode library version evidence: %w", err)
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin library version: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if version.Kind == governance.LibraryVersionProductionEffective {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE library_versions SET superseded_at=$2
|
||||
WHERE scope=$1 AND kind='production_effective' AND superseded_at IS NULL`, version.Scope, version.CreatedAt.UTC()); err != nil {
|
||||
return fmt.Errorf("supersede library version: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id, evidence, created_by, created_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
version.ID, version.Scope, string(version.Kind), version.SnapshotID, version.VersionLabel,
|
||||
version.SourceJobID, evidence, version.CreatedBy, version.CreatedAt.UTC()); err != nil {
|
||||
return fmt.Errorf("record library version: %w", err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (s *Store) LibraryVersions(ctx context.Context, scope string, limit int) ([]governance.LibraryVersion, error) {
|
||||
if limit < 1 || limit > 100 {
|
||||
return nil, errors.New("library version limit out of range")
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, scope, kind, snapshot_id, version_label, source_job_id, evidence, created_by, created_at, superseded_at
|
||||
FROM library_versions WHERE scope=$1 ORDER BY created_at DESC, id DESC LIMIT $2`, scope, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list library versions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make([]governance.LibraryVersion, 0, limit)
|
||||
for rows.Next() {
|
||||
var item governance.LibraryVersion
|
||||
var kind string
|
||||
var evidence []byte
|
||||
if err := rows.Scan(&item.ID, &item.Scope, &kind, &item.SnapshotID, &item.VersionLabel, &item.SourceJobID,
|
||||
&evidence, &item.CreatedBy, &item.CreatedAt, &item.SupersededAt); err != nil {
|
||||
return nil, fmt.Errorf("scan library version: %w", err)
|
||||
}
|
||||
item.Kind = governance.LibraryVersionKind(kind)
|
||||
if len(evidence) > 0 {
|
||||
_ = json.Unmarshal(evidence, &item.Evidence)
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
DROP TABLE IF EXISTS library_job_stages;
|
||||
DROP TABLE IF EXISTS library_jobs;
|
||||
DROP TABLE IF EXISTS library_versions;
|
||||
@@ -0,0 +1,57 @@
|
||||
-- 整库版本与作业:把"测试封存 / 生产生效 / 生产预备份"三种版本与持久作业分开记录。
|
||||
-- 现有 jobs 表是队列与租约;这里记录业务身份、阶段与结果,重启后仍可读。
|
||||
CREATE TABLE library_versions (
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
scope VARCHAR(16) NOT NULL CHECK (scope IN ('internal','external')),
|
||||
kind VARCHAR(24) NOT NULL CHECK (kind IN ('test_sealed','production_effective','production_prebackup')),
|
||||
snapshot_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
version_label VARCHAR(64) NOT NULL DEFAULT '',
|
||||
source_job_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
evidence JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_by VARCHAR(128) NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
superseded_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX library_versions_scope_kind ON library_versions(scope, kind, created_at DESC, id DESC);
|
||||
-- 每个目标只有一个"当前生产生效版本";写入新版本时旧的必须先被 supersede。
|
||||
CREATE UNIQUE INDEX library_versions_effective ON library_versions(scope)
|
||||
WHERE kind = 'production_effective' AND superseded_at IS NULL;
|
||||
|
||||
CREATE TABLE library_jobs (
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
scope VARCHAR(16) NOT NULL CHECK (scope IN ('internal','external')),
|
||||
kind VARCHAR(16) NOT NULL CHECK (kind IN ('seal','sync')),
|
||||
state VARCHAR(16) NOT NULL CHECK (state IN ('pending','running','succeeded','failed','cancelled')),
|
||||
stage VARCHAR(32) NOT NULL DEFAULT '',
|
||||
idempotency_key VARCHAR(255) NOT NULL,
|
||||
request_digest VARCHAR(64) NOT NULL,
|
||||
request JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
queue_job_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
snapshot_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
target_snapshot_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
actor VARCHAR(128) NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
recovery VARCHAR(255) NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL,
|
||||
finished_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE UNIQUE INDEX library_jobs_idempotency ON library_jobs(idempotency_key);
|
||||
-- 同一目标同一类作业只能有一个在飞:互斥由数据库保证,不依赖前端或单 worker 部署。
|
||||
CREATE UNIQUE INDEX library_jobs_active ON library_jobs(scope, kind)
|
||||
WHERE state IN ('pending','running');
|
||||
CREATE INDEX library_jobs_recent ON library_jobs(scope, created_at DESC, id DESC);
|
||||
|
||||
CREATE TABLE library_job_stages (
|
||||
job_id VARCHAR(64) NOT NULL REFERENCES library_jobs(id) ON DELETE CASCADE,
|
||||
stage VARCHAR(32) NOT NULL,
|
||||
state VARCHAR(16) NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','running','succeeded','failed','skipped')),
|
||||
-- NULL 表示"总量未知":绝不能当成 0,也不据此算百分比。
|
||||
total BIGINT,
|
||||
completed BIGINT,
|
||||
bytes_total BIGINT,
|
||||
bytes_done BIGINT,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
updated_at TIMESTAMPTZ NOT NULL,
|
||||
PRIMARY KEY (job_id, stage)
|
||||
);
|
||||
Reference in New Issue
Block a user