98294b9583
build / build (push) Failing after 24s
why:审计 19 项(12 P1 / 7 P2)来自同一批根因,逐点试错会把协议撕成十几份:
· 意图与执行混为一谈(F01/F15):`library.sync:<scope>:<快照>` 是终身幂等键,
A→B→A 的第三次发布拿到第一次的历史成功作业,什么都不做;
· 版本行既是实体又是事件(F02/F09):主键只由 (kind,scope,快照) 决定、写入是
upsert,第二个作业覆盖 `source_job_id`/`evidence`/`created_at`;
· 接管没有代次(F13):两个 worker 都能认领同一 pending 作业,失租的旧执行还能
写业务状态、释放别人的维护锁;
· 早退与终态没有统一政策(F03/F04/F05/F06):半写窗口的前置检查失败会重新开门、
服务起不来也照样开入口、journal 终态后收尾不可重建、恢复成功不回写 current;
· 数据身份与准入各写一套(F07/F08/F14/F17):接管丢了复用语义、全库搬运不改
location_hash、通用入口可造 library.* 作业、目标模型从没被问过;
· 存储/容量/UI/CI 判据各自为政(F10/F11/F12/F16/F18/F19)。
what:按批准的一套协议集中实施(详见 handoffs/release-consolidated-implementation/
current-contract.md):
· operation(一次明确意图)/ attempt(epoch 代次)/ 不可变版本事件 / current 指针;
治理库迁移 76,只动治理库,不碰 WeKnora Schema/源码/镜像;
· 所有业务状态、阶段、版本事件、维护锁写入带 epoch 守卫(失租代次写不进去);
心跳失租立刻取消 handler;
· 半写窗口早退保持关闭;服务未全部健康绝不开入口;journal 终态可从持久事实重建
收尾;恢复成功登记 recovery 事件并推进 current;
· worker 在动维护前核对业务行/kind/scope/队列关联;通用入口拒绝 library.*;
· 全库搬运重算 location_hash 并与文档级导入共用实现;接管消费写集合的复用决定;
发布前核目标实时模型契约、容量门禁前置;
· 存储保护闭包改为全量依赖、类型只认显式登记、保护未知则不出可删候选;
· 页面核 HTTP 状态与结构、恢复纳入轮询、Wiki 差异覆盖标题/摘要/父级。
how:`make verify` rc=0(16 项必需检查全绿,含 go test 全量与前端 485 例);
`go vet ./...` rc=0;`npx tsc -p tsconfig.json --noEmit` rc=0;逐项负例见各包测试
(F03/F04/F08/F12/F13/F14/F16/F18/F19 都有先红后绿记录)。
未执行:治理库迁移到任何已有库、真实 PG/ES/S3/WeKnora 集成、故障注入、部署。
321 lines
14 KiB
Go
321 lines
14 KiB
Go
package postgres
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// 发布执行身份:operation / attempt(epoch)的持久化。
|
||
//
|
||
// 一条铁律写在这里,别处不再重复实现:**谁写业务状态,就必须证明自己是当前代次**。
|
||
// epoch 守卫读 library_jobs.attempt_epoch/lease_owner,与 BeginLibraryAttempt 的激活
|
||
// 是同一把锁;失租代次的所有治理写入都会被拒绝,而不是靠调用方自觉。
|
||
|
||
// ErrLibraryAttemptHeld:同一作业还有未过期的租约。第二个执行者必须放弃,
|
||
// 不能"再认领一次"——那正是 F13 的双 claim。
|
||
var ErrLibraryAttemptHeld = errors.New("library job lease is held by another attempt")
|
||
|
||
// EnsureLibraryOperation 幂等创建/读取一次意图。
|
||
// 同一个 (scope,kind,operation_key) + 同一个 request_digest ⇒ 同一次意图(重放);
|
||
// 摘要不一致 ⇒ 冲突,绝不把新意图塞进旧 operation。
|
||
func (s *Store) EnsureLibraryOperation(ctx context.Context, op governance.LibraryOperation) (governance.LibraryOperation, bool, error) {
|
||
if err := governance.ValidateLibraryOperation(op); err != nil {
|
||
return governance.LibraryOperation{}, false, err
|
||
}
|
||
var id string
|
||
err := s.db.QueryRow(ctx, `
|
||
INSERT INTO library_operations (id, scope, kind, target_snapshot_id, request_digest, operation_key, actor, created_at)
|
||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
|
||
ON CONFLICT (scope, kind, operation_key) DO NOTHING
|
||
RETURNING id`, op.ID, op.Scope, string(op.Kind), op.TargetSnapshotID, op.RequestDigest,
|
||
op.OperationKey, op.Actor, op.CreatedAt.UTC()).Scan(&id)
|
||
if err == nil {
|
||
return op, true, nil
|
||
}
|
||
if !errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryOperation{}, false, fmt.Errorf("create library operation: %w", err)
|
||
}
|
||
existing, loadErr := s.libraryOperationByKey(ctx, op.Scope, op.Kind, op.OperationKey)
|
||
if loadErr != nil {
|
||
return governance.LibraryOperation{}, false, loadErr
|
||
}
|
||
if existing.RequestDigest != op.RequestDigest {
|
||
return existing, false, governance.ErrLibraryOperationConflict
|
||
}
|
||
return existing, false, nil
|
||
}
|
||
|
||
func (s *Store) libraryOperationByKey(ctx context.Context, scope string, kind governance.LibraryJobKind, key string) (governance.LibraryOperation, error) {
|
||
var id string
|
||
if err := s.db.QueryRow(ctx, `
|
||
SELECT id FROM library_operations WHERE scope=$1 AND kind=$2 AND operation_key=$3`,
|
||
scope, string(kind), key).Scan(&id); err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryOperation{}, errors.New("library operation not found")
|
||
}
|
||
return governance.LibraryOperation{}, fmt.Errorf("load library operation: %w", err)
|
||
}
|
||
return s.LibraryOperation(ctx, id)
|
||
}
|
||
|
||
func (s *Store) LibraryOperation(ctx context.Context, id string) (governance.LibraryOperation, error) {
|
||
var op governance.LibraryOperation
|
||
var kind string
|
||
err := s.db.QueryRow(ctx, `
|
||
SELECT id, scope, kind, target_snapshot_id, request_digest, operation_key, actor, created_at
|
||
FROM library_operations WHERE id=$1`, id).Scan(&op.ID, &op.Scope, &kind, &op.TargetSnapshotID,
|
||
&op.RequestDigest, &op.OperationKey, &op.Actor, &op.CreatedAt)
|
||
if err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryOperation{}, err
|
||
}
|
||
return governance.LibraryOperation{}, fmt.Errorf("load library operation: %w", err)
|
||
}
|
||
op.Kind = governance.LibraryJobKind(kind)
|
||
return op, nil
|
||
}
|
||
|
||
// OperationAnchors 读不可变首次锚点。
|
||
func (s *Store) OperationAnchors(ctx context.Context, operationID string) (governance.LibraryAnchors, bool, error) {
|
||
var a governance.LibraryAnchors
|
||
err := s.db.QueryRow(ctx, `
|
||
SELECT first_backup_snapshot_id, first_backup_bytes, first_backup_at, writeset_digest, writeset_rows,
|
||
target_fingerprint, anchors_at
|
||
FROM library_operations WHERE id=$1`, operationID).Scan(&a.FirstBackupSnapshotID, &a.FirstBackupBytes,
|
||
&a.FirstBackupAt, &a.WriteSetDigest, &a.WriteSetRows, &a.TargetFingerprint, &a.RecordedAt)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryAnchors{}, false, nil
|
||
}
|
||
if err != nil {
|
||
return governance.LibraryAnchors{}, false, fmt.Errorf("load library operation anchors: %w", err)
|
||
}
|
||
return a, a.RecordedAt != nil, nil
|
||
}
|
||
|
||
// RecordOperationAnchors 只**填空**,不改已存在的锚点。
|
||
// 接管/重放会用同一份锚点重复调用;如果新值想覆盖旧值,那说明"首次"被重拍了,
|
||
// 直接报错让调用方停下来(把半写状态当新基线等于毁掉唯一可回滚的那份)。
|
||
func (s *Store) RecordOperationAnchors(ctx context.Context, operationID string, anchors governance.LibraryAnchors, now time.Time) error {
|
||
if operationID == "" {
|
||
return errors.New("record operation anchors requires an operation")
|
||
}
|
||
existing, found, err := s.OperationAnchors(ctx, operationID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if found {
|
||
if existing.FirstBackupSnapshotID != "" && anchors.FirstBackupSnapshotID != "" &&
|
||
existing.FirstBackupSnapshotID != anchors.FirstBackupSnapshotID {
|
||
return fmt.Errorf("operation %s 的首次预备份是 %s,不能用 %s 覆盖",
|
||
operationID, existing.FirstBackupSnapshotID, anchors.FirstBackupSnapshotID)
|
||
}
|
||
if existing.WriteSetDigest != "" && anchors.WriteSetDigest != "" &&
|
||
existing.WriteSetDigest != anchors.WriteSetDigest {
|
||
return fmt.Errorf("operation %s 的首次写集合是 %s,不能用 %s 覆盖",
|
||
operationID, existing.WriteSetDigest, anchors.WriteSetDigest)
|
||
}
|
||
}
|
||
_, err = s.db.Exec(ctx, `
|
||
UPDATE library_operations SET
|
||
first_backup_snapshot_id = COALESCE(first_backup_snapshot_id, $2),
|
||
first_backup_bytes = COALESCE(first_backup_bytes, $3),
|
||
first_backup_at = COALESCE(first_backup_at, $4),
|
||
writeset_digest = COALESCE(writeset_digest, $5),
|
||
writeset_rows = COALESCE(writeset_rows, $6),
|
||
target_fingerprint = COALESCE(target_fingerprint, $7),
|
||
anchors_at = COALESCE(anchors_at, $8)
|
||
WHERE id=$1`, operationID, nullIfEmpty(anchors.FirstBackupSnapshotID), anchors.FirstBackupBytes,
|
||
anchors.FirstBackupAt, nullIfEmpty(anchors.WriteSetDigest), anchors.WriteSetRows,
|
||
nullIfEmpty(anchors.TargetFingerprint), now.UTC())
|
||
if err != nil {
|
||
return fmt.Errorf("record library operation anchors: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// BeginLibraryAttempt 取得这个作业的执行权并激活一个新代次。
|
||
//
|
||
// 三件事必须在同一个事务里:判租约(谁能执行)→ 递增 epoch(谁是当前代次)→
|
||
// 写作业行的租约(守卫读它)。分成三步就会出现"两个 worker 都认为自己是当前代次"。
|
||
func (s *Store) BeginLibraryAttempt(ctx context.Context, jobID, owner string, lease time.Duration, now time.Time) (governance.LibraryAttempt, error) {
|
||
if jobID == "" || owner == "" {
|
||
return governance.LibraryAttempt{}, errors.New("begin library attempt requires job and owner")
|
||
}
|
||
if lease <= 0 {
|
||
lease = 2 * time.Minute
|
||
}
|
||
tx, err := s.db.Begin(ctx)
|
||
if err != nil {
|
||
return governance.LibraryAttempt{}, fmt.Errorf("begin library attempt: %w", err)
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
|
||
var operationID, leaseOwner, queueJobID string
|
||
var epoch int
|
||
var leaseUntil *time.Time
|
||
err = tx.QueryRow(ctx, `
|
||
SELECT operation_id, attempt_epoch, lease_owner, lease_until, queue_job_id
|
||
FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan(&operationID, &epoch, &leaseOwner, &leaseUntil, &queueJobID)
|
||
if err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return governance.LibraryAttempt{}, fmt.Errorf("library job %s not found", jobID)
|
||
}
|
||
return governance.LibraryAttempt{}, fmt.Errorf("load library job for attempt: %w", err)
|
||
}
|
||
// 同一 owner 重入是幂等(重试/队列重投);别人持有未过期租约就必须让路。
|
||
if leaseOwner != "" && leaseOwner != owner && leaseUntil != nil && leaseUntil.After(now.UTC()) {
|
||
return governance.LibraryAttempt{}, ErrLibraryAttemptHeld
|
||
}
|
||
nextEpoch := epoch + 1
|
||
if nextEpoch < 1 {
|
||
nextEpoch = 1
|
||
}
|
||
// 代次必须按 operation 单调递增,而不是按作业行:同一个意图的重试/接管会**新开作业行**,
|
||
// 若按作业行各自从 1 数起,旧代次(1)与新代次(1)就分不出先后,fencing 失效。
|
||
if operationID != "" {
|
||
var maxEpoch *int
|
||
if err := tx.QueryRow(ctx, `SELECT MAX(epoch) FROM library_attempts WHERE operation_id=$1`,
|
||
operationID).Scan(&maxEpoch); err != nil {
|
||
return governance.LibraryAttempt{}, fmt.Errorf("load library attempt epoch: %w", err)
|
||
}
|
||
next := 1
|
||
if maxEpoch != nil && *maxEpoch >= next {
|
||
next = *maxEpoch + 1
|
||
}
|
||
if nextEpoch < next {
|
||
nextEpoch = next
|
||
}
|
||
}
|
||
attempt := governance.LibraryAttempt{
|
||
OperationID: operationID, Epoch: nextEpoch, JobID: jobID, QueueJobID: queueJobID,
|
||
Owner: owner, State: governance.LibraryAttemptRunning, StartedAt: now.UTC(),
|
||
}
|
||
until := now.Add(lease).UTC()
|
||
attempt.LeaseUntil = &until
|
||
if operationID != "" {
|
||
// 一个代次只对应一个作业行:重复 Begin(同 job)不再新增谱系。
|
||
if _, err := tx.Exec(ctx, `
|
||
INSERT INTO library_attempts (operation_id, epoch, job_id, queue_job_id, owner, state, started_at, heartbeat_at, lease_until)
|
||
VALUES ($1,$2,$3,$4,$5,'running',$6,$6,$7)
|
||
ON CONFLICT (job_id) DO UPDATE SET
|
||
owner=EXCLUDED.owner, state='running', lease_until=EXCLUDED.lease_until, heartbeat_at=EXCLUDED.heartbeat_at`,
|
||
operationID, nextEpoch, jobID, queueJobID, owner, now.UTC(), until); err != nil {
|
||
return governance.LibraryAttempt{}, fmt.Errorf("record library attempt: %w", err)
|
||
}
|
||
}
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE library_jobs SET attempt_epoch=$2, lease_owner=$3, lease_until=$4, updated_at=$5 WHERE id=$1`,
|
||
jobID, nextEpoch, owner, until, now.UTC()); err != nil {
|
||
return governance.LibraryAttempt{}, fmt.Errorf("activate library attempt: %w", err)
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return governance.LibraryAttempt{}, fmt.Errorf("commit library attempt: %w", err)
|
||
}
|
||
return attempt, nil
|
||
}
|
||
|
||
// FinishLibraryAttempt 落本次代次的终态。旧代次不能结束别人的代次:
|
||
// 只有 (operation, epoch) 与 owner 都匹配才写。
|
||
func (s *Store) FinishLibraryAttempt(ctx context.Context, attempt governance.LibraryAttempt, state governance.LibraryAttemptState, outcome string, now time.Time) error {
|
||
if attempt.OperationID == "" {
|
||
return nil // 旧作业没有 operation:没有代次可结
|
||
}
|
||
tag, err := s.db.Exec(ctx, `
|
||
UPDATE library_attempts SET state=$4, finished_at=$5, outcome=$6, lease_until=NULL
|
||
WHERE operation_id=$1 AND epoch=$2 AND owner=$3`,
|
||
attempt.OperationID, attempt.Epoch, attempt.Owner, string(state), now.UTC(), outcome)
|
||
if err != nil {
|
||
return fmt.Errorf("finish library attempt: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return governance.ErrLibraryEpochStale
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// LibraryAttempts 返回一次 operation 的执行谱系(按代次升序):接管/恢复排查读它。
|
||
func (s *Store) LibraryAttempts(ctx context.Context, operationID string) ([]governance.LibraryAttempt, error) {
|
||
rows, err := s.db.Query(ctx, `
|
||
SELECT operation_id, epoch, job_id, queue_job_id, owner, state, started_at, heartbeat_at, lease_until, finished_at, outcome
|
||
FROM library_attempts WHERE operation_id=$1 ORDER BY epoch`, operationID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list library attempts: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
items := make([]governance.LibraryAttempt, 0, 4)
|
||
for rows.Next() {
|
||
var item governance.LibraryAttempt
|
||
var state string
|
||
if err := rows.Scan(&item.OperationID, &item.Epoch, &item.JobID, &item.QueueJobID, &item.Owner, &state,
|
||
&item.StartedAt, &item.HeartbeatAt, &item.LeaseUntil, &item.FinishedAt, &item.Outcome); err != nil {
|
||
return nil, fmt.Errorf("scan library attempt: %w", err)
|
||
}
|
||
item.State = governance.LibraryAttemptState(state)
|
||
items = append(items, item)
|
||
}
|
||
return items, rows.Err()
|
||
}
|
||
|
||
// LatestLibraryAttempt 取当前代次(无则 false)。
|
||
func (s *Store) LatestLibraryAttempt(ctx context.Context, operationID string) (governance.LibraryAttempt, bool, error) {
|
||
items, err := s.LibraryAttempts(ctx, operationID)
|
||
if err != nil || len(items) == 0 {
|
||
return governance.LibraryAttempt{}, false, err
|
||
}
|
||
return items[len(items)-1], true, nil
|
||
}
|
||
|
||
// RenewLibraryAttempt 续租:失租(0 行)即返回 ErrLibraryEpochStale,调用方必须停手。
|
||
func (s *Store) RenewLibraryAttempt(ctx context.Context, attempt governance.LibraryAttempt, lease time.Duration, now time.Time) error {
|
||
until := now.Add(lease).UTC()
|
||
tag, err := s.db.Exec(ctx, `
|
||
UPDATE library_attempts SET heartbeat_at=$4, lease_until=$5
|
||
WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state='running'`,
|
||
attempt.OperationID, attempt.Epoch, attempt.Owner, now.UTC(), until)
|
||
if err != nil {
|
||
return fmt.Errorf("renew library attempt: %w", err)
|
||
}
|
||
if tag.RowsAffected() != 1 {
|
||
return governance.ErrLibraryEpochStale
|
||
}
|
||
if _, err := s.db.Exec(ctx, `
|
||
UPDATE library_jobs SET lease_until=$4 WHERE id=$1 AND attempt_epoch=$2 AND lease_owner=$3`,
|
||
attempt.JobID, attempt.Epoch, attempt.Owner, until); err != nil {
|
||
return fmt.Errorf("renew library job lease: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// guardLibraryJobEpoch 在事务内核"这次写入来自当前代次"。守卫失败一律返回
|
||
// ErrLibraryEpochStale,调用方不许把它当成"重试一下就好"。
|
||
func guardLibraryJobEpoch(ctx context.Context, tx pgx.Tx, jobID string, epoch int, owner string) error {
|
||
var currentEpoch int
|
||
var currentOwner string
|
||
err := tx.QueryRow(ctx, `
|
||
SELECT attempt_epoch, lease_owner FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan(¤tEpoch, ¤tOwner)
|
||
if err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return fmt.Errorf("library job %s not found", jobID)
|
||
}
|
||
return fmt.Errorf("load library job for epoch guard: %w", err)
|
||
}
|
||
if currentEpoch != epoch || (owner != "" && currentOwner != owner) {
|
||
return fmt.Errorf("%w(作业 %s:本代次 %d/%s,当前 %d/%s)", governance.ErrLibraryEpochStale,
|
||
jobID, epoch, owner, currentEpoch, currentOwner)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func nullIfEmpty(value string) any {
|
||
if value == "" {
|
||
return nil
|
||
}
|
||
return value
|
||
}
|