3643e9bf82
build / build (push) Successful in 1m20s
删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。 新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下 执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。 预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者; 写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
178 lines
7.1 KiB
Go
178 lines
7.1 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
// LibraryInstanceMaintenance 是"实例级维护"的占用记录:一个目标实例同一时刻
|
|
// 只允许一个窗口,无论它来自 internal 还是 external。
|
|
//
|
|
// 为什么不是 scope 锁:stop/start 的是同一批容器,两个 scope 各持一个 scope 锁
|
|
// 时互相看不见,会同时停/起同一实例——那是假互斥,比没有锁更危险。
|
|
type LibraryInstanceMaintenance struct {
|
|
Instance string `json:"instance"`
|
|
OwnerJobID string `json:"owner_job_id"`
|
|
OwnerScope string `json:"owner_scope"`
|
|
TargetFingerprint string `json:"target_fingerprint"`
|
|
Level string `json:"level"`
|
|
Services []string `json:"services"`
|
|
IngressState string `json:"ingress_state"`
|
|
State string `json:"state"`
|
|
OriginalServiceState string `json:"original_service_state"`
|
|
Note string `json:"note"`
|
|
AcquiredAt time.Time `json:"acquired_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
ReleasedAt *time.Time `json:"released_at,omitempty"`
|
|
}
|
|
|
|
const instanceMaintenanceColumns = `instance, owner_job_id, owner_scope, target_fingerprint, level, services,
|
|
ingress_state, state, original_service_state, note, acquired_at, updated_at, released_at`
|
|
|
|
// AcquireInstanceMaintenance 原子地拿到实例维护权:
|
|
// 已释放的记录可以被新作业接管;仍被持有的只允许原作业重入。
|
|
func (s *Store) AcquireInstanceMaintenance(ctx context.Context, item LibraryInstanceMaintenance, now time.Time) (LibraryInstanceMaintenance, error) {
|
|
level := item.Level
|
|
if level == "" {
|
|
level = "instance"
|
|
}
|
|
ingress := item.IngressState
|
|
if ingress == "" {
|
|
ingress = "unknown"
|
|
}
|
|
services := item.Services
|
|
if services == nil {
|
|
services = []string{}
|
|
}
|
|
raw, err := json.Marshal(services)
|
|
if err != nil {
|
|
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
|
|
}
|
|
row := s.db.QueryRow(ctx, `
|
|
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level, services,
|
|
ingress_state, state, original_service_state, note, acquired_at, updated_at)
|
|
VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$11)
|
|
ON CONFLICT (instance) DO UPDATE SET
|
|
owner_job_id = EXCLUDED.owner_job_id,
|
|
owner_scope = EXCLUDED.owner_scope,
|
|
target_fingerprint = EXCLUDED.target_fingerprint,
|
|
level = EXCLUDED.level,
|
|
services = EXCLUDED.services,
|
|
ingress_state = EXCLUDED.ingress_state,
|
|
state = EXCLUDED.state,
|
|
original_service_state = EXCLUDED.original_service_state,
|
|
note = EXCLUDED.note,
|
|
updated_at = EXCLUDED.updated_at,
|
|
released_at = NULL
|
|
WHERE library_instance_maintenance.released_at IS NOT NULL
|
|
OR library_instance_maintenance.owner_job_id = EXCLUDED.owner_job_id
|
|
RETURNING `+instanceMaintenanceColumns,
|
|
item.Instance, item.OwnerJobID, item.OwnerScope, item.TargetFingerprint, level, string(raw),
|
|
ingress, item.State, item.OriginalServiceState, item.Note, now.UTC())
|
|
acquired, err := scanInstanceMaintenance(row)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return LibraryInstanceMaintenance{}, ErrMaintenanceHeld
|
|
}
|
|
if err != nil {
|
|
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
|
|
}
|
|
return acquired, nil
|
|
}
|
|
|
|
// UpdateInstanceMaintenanceState 只允许持有者改自己的窗口状态。
|
|
func (s *Store) UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
|
|
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state=$3, note=$4, updated_at=$5
|
|
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
|
|
if err != nil {
|
|
return fmt.Errorf("update instance maintenance: %w", err)
|
|
}
|
|
if tag.RowsAffected() == 0 {
|
|
return errors.New("instance maintenance not held by this job")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SetInstanceMaintenanceIngress 记录入口真实状态。
|
|
func (s *Store) SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
|
|
if state != "closed" && state != "open" {
|
|
return errors.New("invalid ingress state")
|
|
}
|
|
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET ingress_state=$3, note=$4, updated_at=$5
|
|
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
|
|
if err != nil {
|
|
return fmt.Errorf("update instance maintenance ingress: %w", err)
|
|
}
|
|
if tag.RowsAffected() == 0 {
|
|
return errors.New("instance maintenance not held by this job")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ReleaseInstanceMaintenance 只有"内容已验证 + 服务与入口恢复"才允许调用。
|
|
func (s *Store) ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error {
|
|
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state='released', released_at=$3, updated_at=$3
|
|
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, now.UTC())
|
|
if err != nil {
|
|
return fmt.Errorf("release instance maintenance: %w", err)
|
|
}
|
|
if tag.RowsAffected() == 0 {
|
|
return errors.New("instance maintenance not held by this job")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// InstanceMaintenanceOf 读当前实例窗口(接口互斥判断与页面展示都用它)。
|
|
func (s *Store) InstanceMaintenanceOf(ctx context.Context, instance string) (*LibraryInstanceMaintenance, error) {
|
|
row := s.db.QueryRow(ctx, `SELECT `+instanceMaintenanceColumns+` FROM library_instance_maintenance WHERE instance=$1`, instance)
|
|
item, err := scanInstanceMaintenance(row)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
// HeldInstanceMaintenance 返回仍然持有(未释放)的实例窗口,供接口在不知道
|
|
// 具体实例名时也拒绝写入。
|
|
func (s *Store) HeldInstanceMaintenance(ctx context.Context) ([]LibraryInstanceMaintenance, error) {
|
|
rows, err := s.db.Query(ctx, `SELECT `+instanceMaintenanceColumns+`
|
|
FROM library_instance_maintenance WHERE released_at IS NULL ORDER BY instance`)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list instance maintenance: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
items := make([]LibraryInstanceMaintenance, 0, 2)
|
|
for rows.Next() {
|
|
item, err := scanInstanceMaintenance(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, item)
|
|
}
|
|
return items, rows.Err()
|
|
}
|
|
|
|
func scanInstanceMaintenance(row pgx.Row) (LibraryInstanceMaintenance, error) {
|
|
var item LibraryInstanceMaintenance
|
|
var raw []byte
|
|
err := row.Scan(&item.Instance, &item.OwnerJobID, &item.OwnerScope, &item.TargetFingerprint, &item.Level,
|
|
&raw, &item.IngressState, &item.State, &item.OriginalServiceState, &item.Note,
|
|
&item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt)
|
|
if err != nil {
|
|
return LibraryInstanceMaintenance{}, err
|
|
}
|
|
if len(raw) > 0 {
|
|
if err := json.Unmarshal(raw, &item.Services); err != nil {
|
|
return LibraryInstanceMaintenance{}, fmt.Errorf("instance maintenance services unreadable: %w", err)
|
|
}
|
|
}
|
|
return item, nil
|
|
}
|