feat(library): 整库发布身份、写集合与收尾边界
build / build (push) Successful in 1m45s

执行时真实时钟与持久授权回读(撤销/过期即停并保留预备份)、写集合写前落盘、
租约与 attempt 身份、恢复队列与原发送身份分离、成功收尾必须释放维护后再登记终态。
This commit is contained in:
姚章浩
2026-11-01 11:35:10 +08:00
parent 5afd8c0140
commit 8d838d9cb4
82 changed files with 4809 additions and 551 deletions
@@ -0,0 +1,68 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"time"
)
func (s *Store) CreateLibraryDispatch(ctx context.Context, job governance.LibraryJob, stages []string, handoff bool, now time.Time) (governance.LibraryJob, bool, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return governance.LibraryJob{}, false, err
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
created, reused, err := NewStore(tx).WithLibraryTarget(s.libraryTargetID).createLibraryDispatch(ctx, job, stages, handoff, now)
if err != nil {
return governance.LibraryJob{}, false, err
}
if err = tx.Commit(ctx); err != nil {
return governance.LibraryJob{}, false, err
}
return created, reused, nil
}
func (s *Store) createLibraryDispatch(ctx context.Context, job governance.LibraryJob, stages []string, handoff bool, now time.Time) (governance.LibraryJob, bool, error) {
created, reused, err := s.CreateLibraryJob(ctx, job, stages, now)
if err != nil {
return created, reused, err
}
if reused && (created.State != governance.LibraryJobPending || created.QueueJobID != "") {
return created, true, nil
}
kind := "library." + string(created.Kind)
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore && created.Kind != governance.LibraryJobBaseline {
return created, reused, errors.New("unsupported library dispatch kind")
}
queueID, kind, raw, err := governance.LibraryQueueRequest(handoff, created.ID, created.Scope, created.TargetSnapshotID, kind)
if err != nil {
return created, reused, err
}
var payload map[string]string
if err = json.Unmarshal(raw, &payload); err != nil {
return created, reused, err
}
queued, err := s.Enqueue(ctx, jobs.EnqueueRequest{ID: queueID, Type: kind, Payload: raw, AvailableAt: now, MaxAttempts: 1, IdempotencyKey: kind + ".queue:" + created.ID}, now)
if err != nil {
return created, reused, err
}
if queued == nil || queued.ID != queueID || queued.Type != kind {
return created, reused, errors.New("queue identity conflict")
}
var actual map[string]string
if json.Unmarshal(queued.Payload, &actual) != nil || len(actual) != len(payload) {
return created, reused, errors.New("queue payload mismatch")
}
for key, value := range payload {
if actual[key] != value {
return created, reused, errors.New("queue payload mismatch")
}
}
if err := s.AttachLibraryQueueJob(ctx, created.ID, queued.ID, now); err != nil {
return created, reused, err
}
created.QueueJobID = queued.ID
return created, reused, nil
}
@@ -0,0 +1,36 @@
package postgres
import (
"context"
"errors"
"time"
)
var ErrLibraryExecutionBusy = errors.New("target already has an active executor")
func (s *Store) WithLibraryExecution(ctx context.Context, instance string, run func(context.Context, func(context.Context) error) error) error {
if instance == "" {
return errors.New("execution instance identity required")
}
// A lease expiry is not proof that the old writer has stopped. Keep a
// shared PG advisory lock until the synchronous callback (including cleanup)
// exits. No deferred unlock may run merely because ctx was cancelled.
tx, err := s.db.Begin(ctx)
if err != nil {
return err
}
defer func() {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
_ = tx.Rollback(cleanup)
}()
var held bool
if err = tx.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended($1,0))`, "library-execution:"+instance).Scan(&held); err != nil {
return err
}
if !held {
return ErrLibraryExecutionBusy
}
ping := func(probe context.Context) error { var one int; return tx.QueryRow(probe, `SELECT 1`).Scan(&one) }
return run(ctx, ping)
}
@@ -430,6 +430,7 @@ WHERE lj.kind IN ('sync','restore','baseline')
OR (lj.state='pending' AND j.state='running' AND j.lease_until IS NOT NULL AND j.lease_until < now())
)
AND lj.target_id=$1
AND NOT EXISTS (SELECT 1 FROM jobs recovery WHERE recovery.type='library.recover' AND recovery.payload->>'window_job_id'=lj.id AND recovery.payload->>'scope'=lj.scope)
ORDER BY lj.created_at`, s.libraryTargetID)
if err != nil {
return nil, fmt.Errorf("list interrupted library jobs: %w", err)
@@ -529,6 +530,7 @@ WHERE id=$1 AND state IN ('running','retry','failed','pending')
func (s *Store) ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error) {
tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET state='pending', updated_at=$2
WHERE id=$1 AND target_id=$3 AND state IN ('running','pending')
AND NOT EXISTS (SELECT 1 FROM jobs recovery WHERE recovery.type='library.recover' AND recovery.payload->>'window_job_id'=library_jobs.id AND recovery.payload->>'scope'=library_jobs.scope)
AND (lease_owner='' OR lease_until IS NULL OR lease_until < $2)`, jobID, now.UTC(), s.libraryTargetID)
if err != nil {
return false, fmt.Errorf("claim interrupted library job: %w", err)
@@ -0,0 +1,143 @@
package postgres
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"strings"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryagent"
"knowledge-governance/internal/librarysnapshot"
)
// BindLegacyLibrarySource adds evidence, not a new snapshot or rewritten origin.
// Both the archived manifest and the original capture response must independently
// identify the configured source. A scope match by itself can never authorize it.
func (s *Store) BindLegacyLibrarySource(ctx context.Context, snapshotID, archiveRoot string, source governance.LibrarySourceIdentity, actor string, now time.Time) error {
sourceID, err := source.ID()
if err != nil {
return err
}
if strings.TrimSpace(actor) == "" || now.IsZero() {
return errors.New("source association requires audited actor and time")
}
archive, err := librarysnapshot.Open(archiveRoot, 256*1024*1024)
if err != nil {
return err
}
defer archive.Close()
manifest, err := archive.Load(snapshotID)
if err != nil {
return err
}
if manifest.Scope != source.Scope || manifest.TenantID != source.TenantID || manifest.KnowledgeBaseID != source.KnowledgeBaseID {
return errors.New("historical manifest differs from configured source")
}
tx, err := s.db.Begin(ctx)
if err != nil {
return err
}
defer func() {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
_ = tx.Rollback(cleanup)
}()
rows, err := tx.Query(ctx, `SELECT id,source_job_id,evidence FROM library_versions WHERE scope=$1 AND snapshot_id=$2 AND kind='test_sealed' AND target_id='' FOR UPDATE`, source.Scope, snapshotID)
if err != nil {
return err
}
type sealed struct {
id, job string
evidence []byte
}
items := []sealed{}
for rows.Next() {
var v sealed
if err = rows.Scan(&v.id, &v.job, &v.evidence); err != nil {
rows.Close()
return err
}
items = append(items, v)
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
if len(items) != 1 || items[0].job == "" {
return errors.New("unique original captured seal required")
}
old := items[0]
commitAndVerify := func() error {
if err := tx.Commit(ctx); err != nil {
return err
}
var verifiedID string
if err := s.db.QueryRow(ctx, `SELECT evidence->>'source_id' FROM library_versions WHERE id=$1 AND scope=$2 AND snapshot_id=$3`, old.id, source.Scope, snapshotID).Scan(&verifiedID); err != nil {
return err
}
if verifiedID != sourceID {
return errors.New("committed source association readback mismatch")
}
return nil
}
var evidence map[string]json.RawMessage
if json.Unmarshal(old.evidence, &evidence) != nil || evidence == nil {
return errors.New("original capture evidence is not an object")
}
var captured struct {
SnapshotID string `json:"snapshot_id"`
Identity struct {
Project string `json:"project"`
App string `json:"source_app"`
Root string `json:"archive_root"`
Fingerprint string `json:"expectation_fingerprint"`
} `json:"agent_identity"`
}
var expected string
if json.Unmarshal(evidence["agent_result"], &captured) != nil || json.Unmarshal(evidence["expectation_fingerprint"], &expected) != nil {
return errors.New("original capture identity evidence missing")
}
identity := captured.Identity
fingerprint := libraryagent.NewExpectation(identity.Project, identity.App, identity.Root).Fingerprint
if captured.SnapshotID != snapshotID || identity.Project != source.Project || identity.App != source.App || identity.Root == "" || expected != fingerprint || identity.Fingerprint != expected {
return errors.New("original capture identity or fingerprint mismatch")
}
if raw, exists := evidence["source_id"]; exists {
var current string
var bound governance.LibrarySourceIdentity
if json.Unmarshal(raw, &current) != nil || current != sourceID || json.Unmarshal(evidence["source_identity"], &bound) != nil || bound != source {
return errors.New("sealed source association cannot be changed")
}
return commitAndVerify()
}
for _, reserved := range []string{"source_identity", "legacy_source_binding"} {
if _, exists := evidence[reserved]; exists {
return errors.New("partial source association cannot be overwritten")
}
}
sum := sha256.Sum256(old.evidence)
addition, err := json.Marshal(map[string]any{"source_id": sourceID, "source_identity": source, "legacy_source_binding": map[string]any{"snapshot_id": snapshotID, "original_event_id": old.id, "original_source_job_id": old.job, "original_evidence_sha256": hex.EncodeToString(sum[:]), "expectation_fingerprint": expected, "actor": actor, "bound_at": now.UTC()}})
if err != nil {
return err
}
tag, err := tx.Exec(ctx, `UPDATE library_versions SET evidence=evidence || $2::jsonb WHERE id=$1 AND NOT (evidence ? 'source_id')`, old.id, addition)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return errors.New("source association changed concurrently")
}
var actual string
if err = tx.QueryRow(ctx, `SELECT evidence->>'source_id' FROM library_versions WHERE id=$1`, old.id).Scan(&actual); err != nil {
return err
}
if actual != sourceID {
return errors.New("source association readback mismatch")
}
return commitAndVerify()
}
@@ -459,23 +459,35 @@ func (s *Store) LatestLibraryAttempt(ctx context.Context, operationID string) (g
// RenewLibraryAttempt 续租:失租(0 行)即返回 ErrLibraryEpochStale,调用方必须停手。
func (s *Store) RenewLibraryAttempt(ctx context.Context, attempt governance.LibraryAttempt, lease time.Duration, now time.Time) error {
until := now.Add(lease).UTC()
tag, err := s.db.Exec(ctx, `
UPDATE library_attempts SET heartbeat_at=$4, lease_until=$5
WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state='running'`,
attempt.OperationID, attempt.Epoch, attempt.Owner, now.UTC(), until)
if lease <= 0 {
return errors.New("positive lease required")
}
tx, err := s.db.Begin(ctx)
if err != nil {
return fmt.Errorf("renew library attempt: %w", err)
return err
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
// Lock job first, exactly like BeginLibraryAttempt. Both leases are one fact;
// never resurrect an expired owner or commit half of a renewal.
tag, err := tx.Exec(ctx, `UPDATE library_jobs SET lease_until=clock_timestamp()+$4::interval
WHERE id=$1 AND attempt_epoch=$2 AND lease_owner=$3 AND lease_until>clock_timestamp()`, attempt.JobID, attempt.Epoch, attempt.Owner, lease.String())
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return governance.ErrLibraryEpochStale
}
if _, err := s.db.Exec(ctx, `
UPDATE library_jobs SET lease_until=$4 WHERE id=$1 AND attempt_epoch=$2 AND lease_owner=$3`,
attempt.JobID, attempt.Epoch, attempt.Owner, until); err != nil {
return fmt.Errorf("renew library job lease: %w", err)
if attempt.OperationID != "" {
tag, err = tx.Exec(ctx, `UPDATE library_attempts SET heartbeat_at=clock_timestamp(),lease_until=clock_timestamp()+$4::interval
WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state='running' AND lease_until>clock_timestamp()`, attempt.OperationID, attempt.Epoch, attempt.Owner, lease.String())
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return governance.ErrLibraryEpochStale
}
}
return nil
return tx.Commit(ctx)
}
// guardLibraryJobEpoch 在事务内核"这次写入来自当前代次"。守卫失败一律返回
@@ -0,0 +1,42 @@
package postgres
import (
"context"
"errors"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
// An arbitrary queue delivery cannot borrow another job's window. Check the
// stored request AND its current delivery lease before any maintenance action.
func (s *Store) ResumeExpiredLibraryRecoveryQueues(ctx context.Context) error {
_, err := s.db.Exec(ctx, `UPDATE jobs q SET state='pending',attempt=0,lease_owner=NULL,lease_until=NULL,available_at=clock_timestamp()
FROM library_jobs j,library_instance_maintenance m
WHERE q.type='library.recover' AND q.state='running' AND q.lease_until<clock_timestamp()
AND j.id=q.payload->>'window_job_id' AND j.scope=q.payload->>'scope' AND j.target_id=$1
AND (j.lease_until IS NULL OR j.lease_until<clock_timestamp())
AND m.owner_job_id=j.id AND m.owner_scope=j.scope AND m.released_at IS NULL
AND q.id='lrec-'||substr(encode(sha256(convert_to(j.id,'UTF8')),'hex'),1,28)
AND q.idempotency_key='library.recover:'||j.id`, s.libraryTargetID)
return err
}
func (s *Store) ValidateLibraryRecoveryQueue(ctx context.Context, q jobs.Job, scope, window string) error {
if q.ID != governance.LibraryRecoveryQueueID(window) || q.Type != "library.recover" {
return errors.New("recovery queue identity mismatch")
}
var valid bool
e := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM jobs q JOIN library_jobs j ON j.id=$2
WHERE q.id=$1 AND q.type='library.recover' AND q.state='running'
AND q.payload=$3::jsonb AND q.payload->>'window_job_id'=j.id AND q.payload->>'scope'=j.scope
AND q.idempotency_key='library.recover:'||j.id
AND q.lease_owner=$4 AND q.attempt=$5 AND q.lease_until>clock_timestamp()
AND j.scope=$6 AND j.target_id=$7 AND j.kind IN ('sync','restore','baseline'))`, q.ID, window, q.Payload, q.LeaseOwner, q.Attempt, scope, s.libraryTargetID).Scan(&valid)
if e != nil {
return e
}
if !valid {
return errors.New("recovery request/source/target/lease mismatch")
}
return nil
}
@@ -0,0 +1,26 @@
package postgres
import (
"context"
"errors"
"knowledge-governance/internal/governance"
)
// Query the source identity before applying LIMIT: target/prebackup events and
// other source instances can never push the candidate out of a history window.
func (s *Store) LibrarySourceCandidate(ctx context.Context, scope, sourceID string) (governance.LibraryVersion, bool, error) {
if sourceID == "" {
return governance.LibraryVersion{}, false, errors.New("source identity required")
}
rows, err := s.db.Query(ctx, `SELECT id,scope,kind,snapshot_id,version_label,source_job_id,operation_id,attempt_epoch,evidence,created_by,created_at,superseded_at
FROM library_versions WHERE scope=$1 AND kind='test_sealed' AND evidence->>'source_id'=$2
ORDER BY created_at DESC,id DESC LIMIT 1`, scope, sourceID)
if err != nil {
return governance.LibraryVersion{}, false, err
}
items, err := scanLibraryVersions(rows, 1)
if err != nil || len(items) == 0 {
return governance.LibraryVersion{}, false, err
}
return items[0], true, nil
}
@@ -33,6 +33,7 @@ func libraryFence(jobIDExpr string, epochParam, ownerParam int) string {
WHERE j.id = %s
AND j.attempt_epoch = $%d
AND j.lease_owner = $%d
AND j.lease_until > clock_timestamp()
AND (
j.operation_id = ''
OR EXISTS (
@@ -41,6 +42,7 @@ func libraryFence(jobIDExpr string, epochParam, ownerParam int) string {
AND a.epoch = j.attempt_epoch
AND a.owner = j.lease_owner
AND a.state IN ('claimed','running')
AND a.lease_until > clock_timestamp()
)
)
)`, jobIDExpr, epochParam, ownerParam)