fix(library): 维护收尾必须按预备份恢复并校验
build / build (push) Successful in 1m20s

删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。
新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下
执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。
预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者;
写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
This commit is contained in:
姚章浩
2026-10-29 00:50:32 +08:00
parent ec737a9a65
commit 3643e9bf82
26 changed files with 1730 additions and 225 deletions
@@ -0,0 +1,177 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
)
// LibraryInstanceMaintenance 是"实例级维护"的占用记录:一个目标实例同一时刻
// 只允许一个窗口,无论它来自 internal 还是 external。
//
// 为什么不是 scope 锁:stop/start 的是同一批容器,两个 scope 各持一个 scope 锁
// 时互相看不见,会同时停/起同一实例——那是假互斥,比没有锁更危险。
type LibraryInstanceMaintenance struct {
Instance string `json:"instance"`
OwnerJobID string `json:"owner_job_id"`
OwnerScope string `json:"owner_scope"`
TargetFingerprint string `json:"target_fingerprint"`
Level string `json:"level"`
Services []string `json:"services"`
IngressState string `json:"ingress_state"`
State string `json:"state"`
OriginalServiceState string `json:"original_service_state"`
Note string `json:"note"`
AcquiredAt time.Time `json:"acquired_at"`
UpdatedAt time.Time `json:"updated_at"`
ReleasedAt *time.Time `json:"released_at,omitempty"`
}
const instanceMaintenanceColumns = `instance, owner_job_id, owner_scope, target_fingerprint, level, services,
ingress_state, state, original_service_state, note, acquired_at, updated_at, released_at`
// AcquireInstanceMaintenance 原子地拿到实例维护权:
// 已释放的记录可以被新作业接管;仍被持有的只允许原作业重入。
func (s *Store) AcquireInstanceMaintenance(ctx context.Context, item LibraryInstanceMaintenance, now time.Time) (LibraryInstanceMaintenance, error) {
level := item.Level
if level == "" {
level = "instance"
}
ingress := item.IngressState
if ingress == "" {
ingress = "unknown"
}
services := item.Services
if services == nil {
services = []string{}
}
raw, err := json.Marshal(services)
if err != nil {
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
}
row := s.db.QueryRow(ctx, `
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level, services,
ingress_state, state, original_service_state, note, acquired_at, updated_at)
VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$11)
ON CONFLICT (instance) DO UPDATE SET
owner_job_id = EXCLUDED.owner_job_id,
owner_scope = EXCLUDED.owner_scope,
target_fingerprint = EXCLUDED.target_fingerprint,
level = EXCLUDED.level,
services = EXCLUDED.services,
ingress_state = EXCLUDED.ingress_state,
state = EXCLUDED.state,
original_service_state = EXCLUDED.original_service_state,
note = EXCLUDED.note,
updated_at = EXCLUDED.updated_at,
released_at = NULL
WHERE library_instance_maintenance.released_at IS NOT NULL
OR library_instance_maintenance.owner_job_id = EXCLUDED.owner_job_id
RETURNING `+instanceMaintenanceColumns,
item.Instance, item.OwnerJobID, item.OwnerScope, item.TargetFingerprint, level, string(raw),
ingress, item.State, item.OriginalServiceState, item.Note, now.UTC())
acquired, err := scanInstanceMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return LibraryInstanceMaintenance{}, ErrMaintenanceHeld
}
if err != nil {
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
}
return acquired, nil
}
// UpdateInstanceMaintenanceState 只允许持有者改自己的窗口状态。
func (s *Store) UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state=$3, note=$4, updated_at=$5
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
if err != nil {
return fmt.Errorf("update instance maintenance: %w", err)
}
if tag.RowsAffected() == 0 {
return errors.New("instance maintenance not held by this job")
}
return nil
}
// SetInstanceMaintenanceIngress 记录入口真实状态。
func (s *Store) SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
if state != "closed" && state != "open" {
return errors.New("invalid ingress state")
}
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET ingress_state=$3, note=$4, updated_at=$5
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
if err != nil {
return fmt.Errorf("update instance maintenance ingress: %w", err)
}
if tag.RowsAffected() == 0 {
return errors.New("instance maintenance not held by this job")
}
return nil
}
// ReleaseInstanceMaintenance 只有"内容已验证 + 服务与入口恢复"才允许调用。
func (s *Store) ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state='released', released_at=$3, updated_at=$3
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, now.UTC())
if err != nil {
return fmt.Errorf("release instance maintenance: %w", err)
}
if tag.RowsAffected() == 0 {
return errors.New("instance maintenance not held by this job")
}
return nil
}
// InstanceMaintenanceOf 读当前实例窗口(接口互斥判断与页面展示都用它)。
func (s *Store) InstanceMaintenanceOf(ctx context.Context, instance string) (*LibraryInstanceMaintenance, error) {
row := s.db.QueryRow(ctx, `SELECT `+instanceMaintenanceColumns+` FROM library_instance_maintenance WHERE instance=$1`, instance)
item, err := scanInstanceMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
// HeldInstanceMaintenance 返回仍然持有(未释放)的实例窗口,供接口在不知道
// 具体实例名时也拒绝写入。
func (s *Store) HeldInstanceMaintenance(ctx context.Context) ([]LibraryInstanceMaintenance, error) {
rows, err := s.db.Query(ctx, `SELECT `+instanceMaintenanceColumns+`
FROM library_instance_maintenance WHERE released_at IS NULL ORDER BY instance`)
if err != nil {
return nil, fmt.Errorf("list instance maintenance: %w", err)
}
defer rows.Close()
items := make([]LibraryInstanceMaintenance, 0, 2)
for rows.Next() {
item, err := scanInstanceMaintenance(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func scanInstanceMaintenance(row pgx.Row) (LibraryInstanceMaintenance, error) {
var item LibraryInstanceMaintenance
var raw []byte
err := row.Scan(&item.Instance, &item.OwnerJobID, &item.OwnerScope, &item.TargetFingerprint, &item.Level,
&raw, &item.IngressState, &item.State, &item.OriginalServiceState, &item.Note,
&item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt)
if err != nil {
return LibraryInstanceMaintenance{}, err
}
if len(raw) > 0 {
if err := json.Unmarshal(raw, &item.Services); err != nil {
return LibraryInstanceMaintenance{}, fmt.Errorf("instance maintenance services unreadable: %w", err)
}
}
return item, nil
}
@@ -71,3 +71,45 @@ func TestLibraryMaintenanceOwnershipAcrossRelease(t *testing.T) {
t.Fatalf("released window not re-acquired cleanly: %+v", second)
}
}
// 实例级占用必须跨 scope 互斥:internal 持有期间 external 也进不来。
// (旧实现按 scope 建行,两个 scope 各持一个锁,会同时停/起同一个实例。)
func TestInstanceMaintenanceExcludesAcrossScopes(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatalf("connect: %v", err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("migrate: %v", err)
}
store := NewStore(pool)
now := time.Now().UTC()
base := LibraryInstanceMaintenance{Instance: "kgmaint-test", OwnerJobID: "job-internal",
OwnerScope: "internal", TargetFingerprint: "fp", Level: "instance", Services: []string{"app"},
State: "acquired", Note: "test"}
if _, err := store.AcquireInstanceMaintenance(ctx, base, now); err != nil {
t.Fatalf("first acquire: %v", err)
}
external := base
external.OwnerJobID = "job-external"
external.OwnerScope = "external"
if _, err := store.AcquireInstanceMaintenance(ctx, external, now.Add(time.Second)); !errors.Is(err, ErrMaintenanceHeld) {
t.Fatalf("external scope entered while internal held the instance: err=%v", err)
}
if err := store.ReleaseInstanceMaintenance(ctx, "kgmaint-test", "job-internal", now.Add(2*time.Second)); err != nil {
t.Fatalf("release: %v", err)
}
if _, err := store.AcquireInstanceMaintenance(ctx, external, now.Add(3*time.Second)); err != nil {
t.Fatalf("external scope could not take over the released instance: %v", err)
}
held, err := store.HeldInstanceMaintenance(ctx)
if err != nil || len(held) != 1 || held[0].OwnerScope != "external" {
t.Fatalf("held=%+v err=%v", held, err)
}
}
@@ -0,0 +1 @@
DROP TABLE IF EXISTS library_instance_maintenance;
@@ -0,0 +1,21 @@
-- 维护占用必须按"实例"互斥,而不是按库 scope。
-- 旧表的 scope 主键允许 internal/external 各持一个窗口,于是两个作业会同时
-- 停/起同一个实例的写入者——那是最危险的一种假互斥。
CREATE TABLE library_instance_maintenance (
instance VARCHAR(128) PRIMARY KEY,
owner_job_id VARCHAR(128) NOT NULL,
owner_scope VARCHAR(16) NOT NULL CHECK (owner_scope IN ('internal','external')),
target_fingerprint VARCHAR(64) NOT NULL,
level VARCHAR(16) NOT NULL DEFAULT 'instance' CHECK (level IN ('library','instance')),
services JSONB NOT NULL DEFAULT '[]'::jsonb,
ingress_state VARCHAR(16) NOT NULL DEFAULT 'unknown' CHECK (ingress_state IN ('unknown','closed','open')),
state VARCHAR(24) NOT NULL CHECK (state IN ('acquired','applying','verified','released','needs_restore','recovering')),
original_service_state TEXT NOT NULL DEFAULT '',
note TEXT NOT NULL DEFAULT '',
acquired_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL,
released_at TIMESTAMPTZ
);
-- 把已有窗口按 scope 行回填成实例行不可行(历史 target_identity 是自由文本),
-- 这里只保证新窗口从第一刻起就按实例互斥;旧行保留为审计记录。
+13 -8
View File
@@ -11,30 +11,35 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 71 {
t.Fatalf("migration count = %d, want 71", len(migrations))
if len(migrations) != 72 {
t.Fatalf("migration count = %d, want 72", len(migrations))
}
// 72 让维护占用按实例互斥(internal/external 不再各有各的锁)。
if instance := migrations[len(migrations)-1]; instance.Version != 72 ||
!strings.Contains(instance.UpSQL, "library_instance_maintenance") {
t.Fatalf("migration 72 = %#v", instance)
}
// 71 让维护记录能区分实例级维护与仅治理库互斥。
if level := migrations[len(migrations)-1]; level.Version != 71 ||
if level := migrations[len(migrations)-2]; level.Version != 71 ||
!strings.Contains(level.UpSQL, "level") || !strings.Contains(level.UpSQL, "ingress_state") {
t.Fatalf("migration 71 = %#v", level)
}
if maint := migrations[len(migrations)-2]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
if maint := migrations[len(migrations)-3]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
t.Fatalf("migration 70 = %#v", maint)
}
if tail := migrations[len(migrations)-3]; tail.Version != 69 ||
if tail := migrations[len(migrations)-4]; tail.Version != 69 ||
!strings.Contains(tail.UpSQL, "library_jobs_write_target") {
t.Fatalf("migration 69 = %#v", tail)
}
if restore := migrations[len(migrations)-4]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
if restore := migrations[len(migrations)-5]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
t.Fatalf("migration 68 = %#v", restore)
}
if last := migrations[len(migrations)-5]; last.Version != 67 ||
if last := migrations[len(migrations)-6]; last.Version != 67 ||
!strings.Contains(last.UpSQL, "library_release_authorizations") ||
!strings.Contains(last.UpSQL, "params_digest") {
t.Fatalf("migration 67 = %#v", last)
}
if flow := migrations[len(migrations)-6]; flow.Version != 66 ||
if flow := migrations[len(migrations)-7]; flow.Version != 66 ||
!strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") ||
!strings.Contains(flow.UpSQL, "library_jobs_active") {
t.Fatalf("migration 66 = %#v", flow)