package postgres import ( "context" "errors" "fmt" "time" "github.com/jackc/pgx/v5" "knowledge-governance/internal/governance" ) // 发布执行身份:operation / attempt(epoch)的持久化。 // // 一条铁律写在这里,别处不再重复实现:**谁写业务状态,就必须证明自己是当前代次**。 // epoch 守卫读 library_jobs.attempt_epoch/lease_owner,与 BeginLibraryAttempt 的激活 // 是同一把锁;失租代次的所有治理写入都会被拒绝,而不是靠调用方自觉。 // ErrLibraryAttemptHeld:同一作业还有未过期的租约。第二个执行者必须放弃, // 不能"再认领一次"——那正是 F13 的双 claim。 var ErrLibraryAttemptHeld = errors.New("library job lease is held by another attempt") // EnsureLibraryOperation 幂等创建/读取一次意图。 // 同一个 (scope,kind,operation_key) + 同一个 request_digest ⇒ 同一次意图(重放); // 摘要不一致 ⇒ 冲突,绝不把新意图塞进旧 operation。 func (s *Store) EnsureLibraryOperation(ctx context.Context, op governance.LibraryOperation) (governance.LibraryOperation, bool, error) { if err := governance.ValidateLibraryOperation(op); err != nil { return governance.LibraryOperation{}, false, err } var id string err := s.db.QueryRow(ctx, ` INSERT INTO library_operations (id, scope, kind, target_snapshot_id, request_digest, operation_key, actor, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (scope, kind, operation_key) DO NOTHING RETURNING id`, op.ID, op.Scope, string(op.Kind), op.TargetSnapshotID, op.RequestDigest, op.OperationKey, op.Actor, op.CreatedAt.UTC()).Scan(&id) if err == nil { return op, true, nil } if !errors.Is(err, pgx.ErrNoRows) { return governance.LibraryOperation{}, false, fmt.Errorf("create library operation: %w", err) } existing, loadErr := s.libraryOperationByKey(ctx, op.Scope, op.Kind, op.OperationKey) if loadErr != nil { return governance.LibraryOperation{}, false, loadErr } if existing.RequestDigest != op.RequestDigest { return existing, false, governance.ErrLibraryOperationConflict } return existing, false, nil } func (s *Store) libraryOperationByKey(ctx context.Context, scope string, kind governance.LibraryJobKind, key string) (governance.LibraryOperation, error) { var id string if err := s.db.QueryRow(ctx, ` SELECT id FROM library_operations WHERE scope=$1 AND kind=$2 AND operation_key=$3`, scope, string(kind), key).Scan(&id); err != nil { if errors.Is(err, pgx.ErrNoRows) { return governance.LibraryOperation{}, errors.New("library operation not found") } return governance.LibraryOperation{}, fmt.Errorf("load library operation: %w", err) } return s.LibraryOperation(ctx, id) } func (s *Store) LibraryOperation(ctx context.Context, id string) (governance.LibraryOperation, error) { var op governance.LibraryOperation var kind string err := s.db.QueryRow(ctx, ` SELECT id, scope, kind, target_snapshot_id, request_digest, operation_key, actor, created_at FROM library_operations WHERE id=$1`, id).Scan(&op.ID, &op.Scope, &kind, &op.TargetSnapshotID, &op.RequestDigest, &op.OperationKey, &op.Actor, &op.CreatedAt) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return governance.LibraryOperation{}, err } return governance.LibraryOperation{}, fmt.Errorf("load library operation: %w", err) } op.Kind = governance.LibraryJobKind(kind) return op, nil } // OperationAnchors 读不可变首次锚点。 func (s *Store) OperationAnchors(ctx context.Context, operationID string) (governance.LibraryAnchors, bool, error) { var a governance.LibraryAnchors err := s.db.QueryRow(ctx, ` SELECT first_backup_snapshot_id, first_backup_bytes, first_backup_at, writeset_digest, writeset_rows, target_fingerprint, anchors_at FROM library_operations WHERE id=$1`, operationID).Scan(&a.FirstBackupSnapshotID, &a.FirstBackupBytes, &a.FirstBackupAt, &a.WriteSetDigest, &a.WriteSetRows, &a.TargetFingerprint, &a.RecordedAt) if errors.Is(err, pgx.ErrNoRows) { return governance.LibraryAnchors{}, false, nil } if err != nil { return governance.LibraryAnchors{}, false, fmt.Errorf("load library operation anchors: %w", err) } return a, a.RecordedAt != nil, nil } // RecordOperationAnchors 只**填空**,不改已存在的锚点。 // 接管/重放会用同一份锚点重复调用;如果新值想覆盖旧值,那说明"首次"被重拍了, // 直接报错让调用方停下来(把半写状态当新基线等于毁掉唯一可回滚的那份)。 func (s *Store) RecordOperationAnchors(ctx context.Context, operationID string, anchors governance.LibraryAnchors, now time.Time) error { if operationID == "" { return errors.New("record operation anchors requires an operation") } existing, found, err := s.OperationAnchors(ctx, operationID) if err != nil { return err } if found { if existing.FirstBackupSnapshotID != "" && anchors.FirstBackupSnapshotID != "" && existing.FirstBackupSnapshotID != anchors.FirstBackupSnapshotID { return fmt.Errorf("operation %s 的首次预备份是 %s,不能用 %s 覆盖", operationID, existing.FirstBackupSnapshotID, anchors.FirstBackupSnapshotID) } if existing.WriteSetDigest != "" && anchors.WriteSetDigest != "" && existing.WriteSetDigest != anchors.WriteSetDigest { return fmt.Errorf("operation %s 的首次写集合是 %s,不能用 %s 覆盖", operationID, existing.WriteSetDigest, anchors.WriteSetDigest) } } _, err = s.db.Exec(ctx, ` UPDATE library_operations SET first_backup_snapshot_id = COALESCE(first_backup_snapshot_id, $2), first_backup_bytes = COALESCE(first_backup_bytes, $3), first_backup_at = COALESCE(first_backup_at, $4), writeset_digest = COALESCE(writeset_digest, $5), writeset_rows = COALESCE(writeset_rows, $6), target_fingerprint = COALESCE(target_fingerprint, $7), anchors_at = COALESCE(anchors_at, $8) WHERE id=$1`, operationID, nullIfEmpty(anchors.FirstBackupSnapshotID), anchors.FirstBackupBytes, anchors.FirstBackupAt, nullIfEmpty(anchors.WriteSetDigest), anchors.WriteSetRows, nullIfEmpty(anchors.TargetFingerprint), now.UTC()) if err != nil { return fmt.Errorf("record library operation anchors: %w", err) } return nil } // BeginLibraryAttempt 取得这个作业的执行权并激活一个新代次。 // // 三件事必须在同一个事务里:判租约(谁能执行)→ 递增 epoch(谁是当前代次)→ // 写作业行的租约(守卫读它)。分成三步就会出现"两个 worker 都认为自己是当前代次"。 func (s *Store) BeginLibraryAttempt(ctx context.Context, jobID, owner string, lease time.Duration, now time.Time) (governance.LibraryAttempt, error) { if jobID == "" || owner == "" { return governance.LibraryAttempt{}, errors.New("begin library attempt requires job and owner") } if lease <= 0 { lease = 2 * time.Minute } tx, err := s.db.Begin(ctx) if err != nil { return governance.LibraryAttempt{}, fmt.Errorf("begin library attempt: %w", err) } defer func() { _ = tx.Rollback(ctx) }() var operationID, leaseOwner, queueJobID string var epoch int var leaseUntil *time.Time err = tx.QueryRow(ctx, ` SELECT operation_id, attempt_epoch, lease_owner, lease_until, queue_job_id FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan(&operationID, &epoch, &leaseOwner, &leaseUntil, &queueJobID) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return governance.LibraryAttempt{}, fmt.Errorf("library job %s not found", jobID) } return governance.LibraryAttempt{}, fmt.Errorf("load library job for attempt: %w", err) } // 同一 owner 重入是幂等(重试/队列重投);别人持有未过期租约就必须让路。 if leaseOwner != "" && leaseOwner != owner && leaseUntil != nil && leaseUntil.After(now.UTC()) { return governance.LibraryAttempt{}, ErrLibraryAttemptHeld } nextEpoch := epoch + 1 if nextEpoch < 1 { nextEpoch = 1 } // 代次必须按 operation 单调递增,而不是按作业行:同一个意图的重试/接管会**新开作业行**, // 若按作业行各自从 1 数起,旧代次(1)与新代次(1)就分不出先后,fencing 失效。 if operationID != "" { var maxEpoch *int if err := tx.QueryRow(ctx, `SELECT MAX(epoch) FROM library_attempts WHERE operation_id=$1`, operationID).Scan(&maxEpoch); err != nil { return governance.LibraryAttempt{}, fmt.Errorf("load library attempt epoch: %w", err) } next := 1 if maxEpoch != nil && *maxEpoch >= next { next = *maxEpoch + 1 } if nextEpoch < next { nextEpoch = next } } attempt := governance.LibraryAttempt{ OperationID: operationID, Epoch: nextEpoch, JobID: jobID, QueueJobID: queueJobID, Owner: owner, State: governance.LibraryAttemptRunning, StartedAt: now.UTC(), } until := now.Add(lease).UTC() attempt.LeaseUntil = &until if operationID != "" { // 一个代次只对应一个作业行:重复 Begin(同 job)不再新增谱系。 if _, err := tx.Exec(ctx, ` INSERT INTO library_attempts (operation_id, epoch, job_id, queue_job_id, owner, state, started_at, heartbeat_at, lease_until) VALUES ($1,$2,$3,$4,$5,'running',$6,$6,$7) ON CONFLICT (job_id) DO UPDATE SET owner=EXCLUDED.owner, state='running', lease_until=EXCLUDED.lease_until, heartbeat_at=EXCLUDED.heartbeat_at`, operationID, nextEpoch, jobID, queueJobID, owner, now.UTC(), until); err != nil { return governance.LibraryAttempt{}, fmt.Errorf("record library attempt: %w", err) } } if _, err := tx.Exec(ctx, ` UPDATE library_jobs SET attempt_epoch=$2, lease_owner=$3, lease_until=$4, updated_at=$5 WHERE id=$1`, jobID, nextEpoch, owner, until, now.UTC()); err != nil { return governance.LibraryAttempt{}, fmt.Errorf("activate library attempt: %w", err) } if err := tx.Commit(ctx); err != nil { return governance.LibraryAttempt{}, fmt.Errorf("commit library attempt: %w", err) } return attempt, nil } // FinishLibraryAttempt 落本次代次的终态。旧代次不能结束别人的代次: // 只有 (operation, epoch) 与 owner 都匹配才写。 func (s *Store) FinishLibraryAttempt(ctx context.Context, attempt governance.LibraryAttempt, state governance.LibraryAttemptState, outcome string, now time.Time) error { if attempt.OperationID == "" { return nil // 旧作业没有 operation:没有代次可结 } tag, err := s.db.Exec(ctx, ` UPDATE library_attempts SET state=$4, finished_at=$5, outcome=$6, lease_until=NULL WHERE operation_id=$1 AND epoch=$2 AND owner=$3`, attempt.OperationID, attempt.Epoch, attempt.Owner, string(state), now.UTC(), outcome) if err != nil { return fmt.Errorf("finish library attempt: %w", err) } if tag.RowsAffected() != 1 { return governance.ErrLibraryEpochStale } return nil } // LibraryAttempts 返回一次 operation 的执行谱系(按代次升序):接管/恢复排查读它。 func (s *Store) LibraryAttempts(ctx context.Context, operationID string) ([]governance.LibraryAttempt, error) { rows, err := s.db.Query(ctx, ` SELECT operation_id, epoch, job_id, queue_job_id, owner, state, started_at, heartbeat_at, lease_until, finished_at, outcome FROM library_attempts WHERE operation_id=$1 ORDER BY epoch`, operationID) if err != nil { return nil, fmt.Errorf("list library attempts: %w", err) } defer rows.Close() items := make([]governance.LibraryAttempt, 0, 4) for rows.Next() { var item governance.LibraryAttempt var state string if err := rows.Scan(&item.OperationID, &item.Epoch, &item.JobID, &item.QueueJobID, &item.Owner, &state, &item.StartedAt, &item.HeartbeatAt, &item.LeaseUntil, &item.FinishedAt, &item.Outcome); err != nil { return nil, fmt.Errorf("scan library attempt: %w", err) } item.State = governance.LibraryAttemptState(state) items = append(items, item) } return items, rows.Err() } // LatestLibraryAttempt 取当前代次(无则 false)。 func (s *Store) LatestLibraryAttempt(ctx context.Context, operationID string) (governance.LibraryAttempt, bool, error) { items, err := s.LibraryAttempts(ctx, operationID) if err != nil || len(items) == 0 { return governance.LibraryAttempt{}, false, err } return items[len(items)-1], true, nil } // RenewLibraryAttempt 续租:失租(0 行)即返回 ErrLibraryEpochStale,调用方必须停手。 func (s *Store) RenewLibraryAttempt(ctx context.Context, attempt governance.LibraryAttempt, lease time.Duration, now time.Time) error { until := now.Add(lease).UTC() tag, err := s.db.Exec(ctx, ` UPDATE library_attempts SET heartbeat_at=$4, lease_until=$5 WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state='running'`, attempt.OperationID, attempt.Epoch, attempt.Owner, now.UTC(), until) if err != nil { return fmt.Errorf("renew library attempt: %w", err) } if tag.RowsAffected() != 1 { return governance.ErrLibraryEpochStale } if _, err := s.db.Exec(ctx, ` UPDATE library_jobs SET lease_until=$4 WHERE id=$1 AND attempt_epoch=$2 AND lease_owner=$3`, attempt.JobID, attempt.Epoch, attempt.Owner, until); err != nil { return fmt.Errorf("renew library job lease: %w", err) } return nil } // guardLibraryJobEpoch 在事务内核"这次写入来自当前代次"。守卫失败一律返回 // ErrLibraryEpochStale,调用方不许把它当成"重试一下就好"。 func guardLibraryJobEpoch(ctx context.Context, tx pgx.Tx, jobID string, epoch int, owner string) error { var currentEpoch int var currentOwner string err := tx.QueryRow(ctx, ` SELECT attempt_epoch, lease_owner FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan(¤tEpoch, ¤tOwner) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return fmt.Errorf("library job %s not found", jobID) } return fmt.Errorf("load library job for epoch guard: %w", err) } if currentEpoch != epoch || (owner != "" && currentOwner != owner) { return fmt.Errorf("%w(作业 %s:本代次 %d/%s,当前 %d/%s)", governance.ErrLibraryEpochStale, jobID, epoch, owner, currentEpoch, currentOwner) } return nil } func nullIfEmpty(value string) any { if value == "" { return nil } return value }