执行时真实时钟与持久授权回读(撤销/过期即停并保留预备份)、写集合写前落盘、 租约与 attempt 身份、恢复队列与原发送身份分离、成功收尾必须释放维护后再登记终态。
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (s *Store) CreateLibraryDispatch(ctx context.Context, job governance.LibraryJob, stages []string, handoff bool, now time.Time) (governance.LibraryJob, bool, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, false, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
created, reused, err := NewStore(tx).WithLibraryTarget(s.libraryTargetID).createLibraryDispatch(ctx, job, stages, handoff, now)
|
||||
if err != nil {
|
||||
return governance.LibraryJob{}, false, err
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return governance.LibraryJob{}, false, err
|
||||
}
|
||||
return created, reused, nil
|
||||
}
|
||||
func (s *Store) createLibraryDispatch(ctx context.Context, job governance.LibraryJob, stages []string, handoff bool, now time.Time) (governance.LibraryJob, bool, error) {
|
||||
created, reused, err := s.CreateLibraryJob(ctx, job, stages, now)
|
||||
if err != nil {
|
||||
return created, reused, err
|
||||
}
|
||||
if reused && (created.State != governance.LibraryJobPending || created.QueueJobID != "") {
|
||||
return created, true, nil
|
||||
}
|
||||
kind := "library." + string(created.Kind)
|
||||
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore && created.Kind != governance.LibraryJobBaseline {
|
||||
return created, reused, errors.New("unsupported library dispatch kind")
|
||||
}
|
||||
queueID, kind, raw, err := governance.LibraryQueueRequest(handoff, created.ID, created.Scope, created.TargetSnapshotID, kind)
|
||||
if err != nil {
|
||||
return created, reused, err
|
||||
}
|
||||
var payload map[string]string
|
||||
if err = json.Unmarshal(raw, &payload); err != nil {
|
||||
return created, reused, err
|
||||
}
|
||||
queued, err := s.Enqueue(ctx, jobs.EnqueueRequest{ID: queueID, Type: kind, Payload: raw, AvailableAt: now, MaxAttempts: 1, IdempotencyKey: kind + ".queue:" + created.ID}, now)
|
||||
if err != nil {
|
||||
return created, reused, err
|
||||
}
|
||||
if queued == nil || queued.ID != queueID || queued.Type != kind {
|
||||
return created, reused, errors.New("queue identity conflict")
|
||||
}
|
||||
var actual map[string]string
|
||||
if json.Unmarshal(queued.Payload, &actual) != nil || len(actual) != len(payload) {
|
||||
return created, reused, errors.New("queue payload mismatch")
|
||||
}
|
||||
for key, value := range payload {
|
||||
if actual[key] != value {
|
||||
return created, reused, errors.New("queue payload mismatch")
|
||||
}
|
||||
}
|
||||
if err := s.AttachLibraryQueueJob(ctx, created.ID, queued.ID, now); err != nil {
|
||||
return created, reused, err
|
||||
}
|
||||
created.QueueJobID = queued.ID
|
||||
return created, reused, nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrLibraryExecutionBusy = errors.New("target already has an active executor")
|
||||
|
||||
func (s *Store) WithLibraryExecution(ctx context.Context, instance string, run func(context.Context, func(context.Context) error) error) error {
|
||||
if instance == "" {
|
||||
return errors.New("execution instance identity required")
|
||||
}
|
||||
// A lease expiry is not proof that the old writer has stopped. Keep a
|
||||
// shared PG advisory lock until the synchronous callback (including cleanup)
|
||||
// exits. No deferred unlock may run merely because ctx was cancelled.
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = tx.Rollback(cleanup)
|
||||
}()
|
||||
var held bool
|
||||
if err = tx.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended($1,0))`, "library-execution:"+instance).Scan(&held); err != nil {
|
||||
return err
|
||||
}
|
||||
if !held {
|
||||
return ErrLibraryExecutionBusy
|
||||
}
|
||||
ping := func(probe context.Context) error { var one int; return tx.QueryRow(probe, `SELECT 1`).Scan(&one) }
|
||||
return run(ctx, ping)
|
||||
}
|
||||
@@ -430,6 +430,7 @@ WHERE lj.kind IN ('sync','restore','baseline')
|
||||
OR (lj.state='pending' AND j.state='running' AND j.lease_until IS NOT NULL AND j.lease_until < now())
|
||||
)
|
||||
AND lj.target_id=$1
|
||||
AND NOT EXISTS (SELECT 1 FROM jobs recovery WHERE recovery.type='library.recover' AND recovery.payload->>'window_job_id'=lj.id AND recovery.payload->>'scope'=lj.scope)
|
||||
ORDER BY lj.created_at`, s.libraryTargetID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list interrupted library jobs: %w", err)
|
||||
@@ -529,6 +530,7 @@ WHERE id=$1 AND state IN ('running','retry','failed','pending')
|
||||
func (s *Store) ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error) {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET state='pending', updated_at=$2
|
||||
WHERE id=$1 AND target_id=$3 AND state IN ('running','pending')
|
||||
AND NOT EXISTS (SELECT 1 FROM jobs recovery WHERE recovery.type='library.recover' AND recovery.payload->>'window_job_id'=library_jobs.id AND recovery.payload->>'scope'=library_jobs.scope)
|
||||
AND (lease_owner='' OR lease_until IS NULL OR lease_until < $2)`, jobID, now.UTC(), s.libraryTargetID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("claim interrupted library job: %w", err)
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryagent"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// BindLegacyLibrarySource adds evidence, not a new snapshot or rewritten origin.
|
||||
// Both the archived manifest and the original capture response must independently
|
||||
// identify the configured source. A scope match by itself can never authorize it.
|
||||
func (s *Store) BindLegacyLibrarySource(ctx context.Context, snapshotID, archiveRoot string, source governance.LibrarySourceIdentity, actor string, now time.Time) error {
|
||||
sourceID, err := source.ID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(actor) == "" || now.IsZero() {
|
||||
return errors.New("source association requires audited actor and time")
|
||||
}
|
||||
archive, err := librarysnapshot.Open(archiveRoot, 256*1024*1024)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer archive.Close()
|
||||
manifest, err := archive.Load(snapshotID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if manifest.Scope != source.Scope || manifest.TenantID != source.TenantID || manifest.KnowledgeBaseID != source.KnowledgeBaseID {
|
||||
return errors.New("historical manifest differs from configured source")
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = tx.Rollback(cleanup)
|
||||
}()
|
||||
rows, err := tx.Query(ctx, `SELECT id,source_job_id,evidence FROM library_versions WHERE scope=$1 AND snapshot_id=$2 AND kind='test_sealed' AND target_id='' FOR UPDATE`, source.Scope, snapshotID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type sealed struct {
|
||||
id, job string
|
||||
evidence []byte
|
||||
}
|
||||
items := []sealed{}
|
||||
for rows.Next() {
|
||||
var v sealed
|
||||
if err = rows.Scan(&v.id, &v.job, &v.evidence); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
items = append(items, v)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(items) != 1 || items[0].job == "" {
|
||||
return errors.New("unique original captured seal required")
|
||||
}
|
||||
old := items[0]
|
||||
commitAndVerify := func() error {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
var verifiedID string
|
||||
if err := s.db.QueryRow(ctx, `SELECT evidence->>'source_id' FROM library_versions WHERE id=$1 AND scope=$2 AND snapshot_id=$3`, old.id, source.Scope, snapshotID).Scan(&verifiedID); err != nil {
|
||||
return err
|
||||
}
|
||||
if verifiedID != sourceID {
|
||||
return errors.New("committed source association readback mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var evidence map[string]json.RawMessage
|
||||
if json.Unmarshal(old.evidence, &evidence) != nil || evidence == nil {
|
||||
return errors.New("original capture evidence is not an object")
|
||||
}
|
||||
var captured struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Identity struct {
|
||||
Project string `json:"project"`
|
||||
App string `json:"source_app"`
|
||||
Root string `json:"archive_root"`
|
||||
Fingerprint string `json:"expectation_fingerprint"`
|
||||
} `json:"agent_identity"`
|
||||
}
|
||||
var expected string
|
||||
if json.Unmarshal(evidence["agent_result"], &captured) != nil || json.Unmarshal(evidence["expectation_fingerprint"], &expected) != nil {
|
||||
return errors.New("original capture identity evidence missing")
|
||||
}
|
||||
identity := captured.Identity
|
||||
fingerprint := libraryagent.NewExpectation(identity.Project, identity.App, identity.Root).Fingerprint
|
||||
if captured.SnapshotID != snapshotID || identity.Project != source.Project || identity.App != source.App || identity.Root == "" || expected != fingerprint || identity.Fingerprint != expected {
|
||||
return errors.New("original capture identity or fingerprint mismatch")
|
||||
}
|
||||
if raw, exists := evidence["source_id"]; exists {
|
||||
var current string
|
||||
var bound governance.LibrarySourceIdentity
|
||||
if json.Unmarshal(raw, ¤t) != nil || current != sourceID || json.Unmarshal(evidence["source_identity"], &bound) != nil || bound != source {
|
||||
return errors.New("sealed source association cannot be changed")
|
||||
}
|
||||
return commitAndVerify()
|
||||
}
|
||||
for _, reserved := range []string{"source_identity", "legacy_source_binding"} {
|
||||
if _, exists := evidence[reserved]; exists {
|
||||
return errors.New("partial source association cannot be overwritten")
|
||||
}
|
||||
}
|
||||
sum := sha256.Sum256(old.evidence)
|
||||
addition, err := json.Marshal(map[string]any{"source_id": sourceID, "source_identity": source, "legacy_source_binding": map[string]any{"snapshot_id": snapshotID, "original_event_id": old.id, "original_source_job_id": old.job, "original_evidence_sha256": hex.EncodeToString(sum[:]), "expectation_fingerprint": expected, "actor": actor, "bound_at": now.UTC()}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE library_versions SET evidence=evidence || $2::jsonb WHERE id=$1 AND NOT (evidence ? 'source_id')`, old.id, addition)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return errors.New("source association changed concurrently")
|
||||
}
|
||||
var actual string
|
||||
if err = tx.QueryRow(ctx, `SELECT evidence->>'source_id' FROM library_versions WHERE id=$1`, old.id).Scan(&actual); err != nil {
|
||||
return err
|
||||
}
|
||||
if actual != sourceID {
|
||||
return errors.New("source association readback mismatch")
|
||||
}
|
||||
return commitAndVerify()
|
||||
}
|
||||
@@ -459,23 +459,35 @@ func (s *Store) LatestLibraryAttempt(ctx context.Context, operationID string) (g
|
||||
|
||||
// RenewLibraryAttempt 续租:失租(0 行)即返回 ErrLibraryEpochStale,调用方必须停手。
|
||||
func (s *Store) RenewLibraryAttempt(ctx context.Context, attempt governance.LibraryAttempt, lease time.Duration, now time.Time) error {
|
||||
until := now.Add(lease).UTC()
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE library_attempts SET heartbeat_at=$4, lease_until=$5
|
||||
WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state='running'`,
|
||||
attempt.OperationID, attempt.Epoch, attempt.Owner, now.UTC(), until)
|
||||
if lease <= 0 {
|
||||
return errors.New("positive lease required")
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("renew library attempt: %w", err)
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
// Lock job first, exactly like BeginLibraryAttempt. Both leases are one fact;
|
||||
// never resurrect an expired owner or commit half of a renewal.
|
||||
tag, err := tx.Exec(ctx, `UPDATE library_jobs SET lease_until=clock_timestamp()+$4::interval
|
||||
WHERE id=$1 AND attempt_epoch=$2 AND lease_owner=$3 AND lease_until>clock_timestamp()`, attempt.JobID, attempt.Epoch, attempt.Owner, lease.String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return governance.ErrLibraryEpochStale
|
||||
}
|
||||
if _, err := s.db.Exec(ctx, `
|
||||
UPDATE library_jobs SET lease_until=$4 WHERE id=$1 AND attempt_epoch=$2 AND lease_owner=$3`,
|
||||
attempt.JobID, attempt.Epoch, attempt.Owner, until); err != nil {
|
||||
return fmt.Errorf("renew library job lease: %w", err)
|
||||
if attempt.OperationID != "" {
|
||||
tag, err = tx.Exec(ctx, `UPDATE library_attempts SET heartbeat_at=clock_timestamp(),lease_until=clock_timestamp()+$4::interval
|
||||
WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state='running' AND lease_until>clock_timestamp()`, attempt.OperationID, attempt.Epoch, attempt.Owner, lease.String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return governance.ErrLibraryEpochStale
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// guardLibraryJobEpoch 在事务内核"这次写入来自当前代次"。守卫失败一律返回
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
// An arbitrary queue delivery cannot borrow another job's window. Check the
|
||||
// stored request AND its current delivery lease before any maintenance action.
|
||||
func (s *Store) ResumeExpiredLibraryRecoveryQueues(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `UPDATE jobs q SET state='pending',attempt=0,lease_owner=NULL,lease_until=NULL,available_at=clock_timestamp()
|
||||
FROM library_jobs j,library_instance_maintenance m
|
||||
WHERE q.type='library.recover' AND q.state='running' AND q.lease_until<clock_timestamp()
|
||||
AND j.id=q.payload->>'window_job_id' AND j.scope=q.payload->>'scope' AND j.target_id=$1
|
||||
AND (j.lease_until IS NULL OR j.lease_until<clock_timestamp())
|
||||
AND m.owner_job_id=j.id AND m.owner_scope=j.scope AND m.released_at IS NULL
|
||||
AND q.id='lrec-'||substr(encode(sha256(convert_to(j.id,'UTF8')),'hex'),1,28)
|
||||
AND q.idempotency_key='library.recover:'||j.id`, s.libraryTargetID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) ValidateLibraryRecoveryQueue(ctx context.Context, q jobs.Job, scope, window string) error {
|
||||
if q.ID != governance.LibraryRecoveryQueueID(window) || q.Type != "library.recover" {
|
||||
return errors.New("recovery queue identity mismatch")
|
||||
}
|
||||
var valid bool
|
||||
e := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM jobs q JOIN library_jobs j ON j.id=$2
|
||||
WHERE q.id=$1 AND q.type='library.recover' AND q.state='running'
|
||||
AND q.payload=$3::jsonb AND q.payload->>'window_job_id'=j.id AND q.payload->>'scope'=j.scope
|
||||
AND q.idempotency_key='library.recover:'||j.id
|
||||
AND q.lease_owner=$4 AND q.attempt=$5 AND q.lease_until>clock_timestamp()
|
||||
AND j.scope=$6 AND j.target_id=$7 AND j.kind IN ('sync','restore','baseline'))`, q.ID, window, q.Payload, q.LeaseOwner, q.Attempt, scope, s.libraryTargetID).Scan(&valid)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if !valid {
|
||||
return errors.New("recovery request/source/target/lease mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// Query the source identity before applying LIMIT: target/prebackup events and
|
||||
// other source instances can never push the candidate out of a history window.
|
||||
func (s *Store) LibrarySourceCandidate(ctx context.Context, scope, sourceID string) (governance.LibraryVersion, bool, error) {
|
||||
if sourceID == "" {
|
||||
return governance.LibraryVersion{}, false, errors.New("source identity required")
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `SELECT id,scope,kind,snapshot_id,version_label,source_job_id,operation_id,attempt_epoch,evidence,created_by,created_at,superseded_at
|
||||
FROM library_versions WHERE scope=$1 AND kind='test_sealed' AND evidence->>'source_id'=$2
|
||||
ORDER BY created_at DESC,id DESC LIMIT 1`, scope, sourceID)
|
||||
if err != nil {
|
||||
return governance.LibraryVersion{}, false, err
|
||||
}
|
||||
items, err := scanLibraryVersions(rows, 1)
|
||||
if err != nil || len(items) == 0 {
|
||||
return governance.LibraryVersion{}, false, err
|
||||
}
|
||||
return items[0], true, nil
|
||||
}
|
||||
@@ -33,6 +33,7 @@ func libraryFence(jobIDExpr string, epochParam, ownerParam int) string {
|
||||
WHERE j.id = %s
|
||||
AND j.attempt_epoch = $%d
|
||||
AND j.lease_owner = $%d
|
||||
AND j.lease_until > clock_timestamp()
|
||||
AND (
|
||||
j.operation_id = ''
|
||||
OR EXISTS (
|
||||
@@ -41,6 +42,7 @@ func libraryFence(jobIDExpr string, epochParam, ownerParam int) string {
|
||||
AND a.epoch = j.attempt_epoch
|
||||
AND a.owner = j.lease_owner
|
||||
AND a.state IN ('claimed','running')
|
||||
AND a.lease_until > clock_timestamp()
|
||||
)
|
||||
)
|
||||
)`, jobIDExpr, epochParam, ownerParam)
|
||||
|
||||
Reference in New Issue
Block a user