Files
knowledge-governance/internal/platform/postgres/library_instance_maintenance.go
T
姚章浩 a42b101a7b
build / build (push) Successful in 1m25s
fix(library): 基线崩溃续作与维护原状态保留
父审发现三处会让目标卡死或状态对不上实际入口,逐条修:

1. 登记成功但没来得及收尾:崩溃重入会撞上"已有生效版本"直接失败,
   目标留在维护里、占用也不释放。现在按「本作业+同一快照」识别为自己的
   版本,只做收尾(恢复服务/开入口/释放两把占用),不重新捕获、不重写;
   别人的版本仍然拒绝。捕获完成但登记失败则明确失败并写"未对目标写入"。
2. 重入覆盖原运行状态:崩溃时服务已被停住,再读一次只会读到 stopped,
   写回去会让恢复阶段永远不启动写入者。AcquireInstance/LibraryMaintenance
   在 owner 相同的重入时保留已持久化原状态;新作业接管已释放窗口用自己的观察。
   真实 PostgreSQL 集成用例覆盖(先红后绿)。
3. 版本登记移进维护窗口:捕获/校验/登记要么一起成立、要么目标原样恢复;
   记录失败时窗口已开放,治理状态同步释放,不再"显示维护中、实际已开"。
   索引映射读不到时拒绝登记,不再用空摘要写证据。

顺带:
- 基线窗口统一收尾抽成 settleBaselineWindow,四种结局各有测试(成功/健康失败/
  held 保持维护/已恢复则释放),并用真实 DB 用例证明 baseline 与 sync/restore
  共用写互斥、幂等重试不产生第二个作业。
- 新增 cmd/librarydrill:维护隔离 → 整库预备份 → 一致性校验 → canary 漂移检测
  → 按预备份恢复 → 逐类复核 → 按原状态恢复服务。用与 worker 相同的 LIBRARY_*
  配置,绝不触碰封存源。
- 目标/预备份/维护栈装配收成 internal/librarystack 一份(worker 与演练共用)。

验证:go build/vet/test 全过;TEST_DATABASE_URL 指向临时 PostgreSQL 跑
TestMaintenanceReentryKeepsOriginalServiceState(修复前失败:got app=stopped)
与 TestLibraryWriteMutexCoversBaselineJobs;本地门禁 15/15。
2026-10-29 19:08:58 +08:00

195 lines
8.0 KiB
Go

package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
)
// LibraryInstanceMaintenance 是"实例级维护"的占用记录:一个目标实例同一时刻
// 只允许一个窗口,无论它来自 internal 还是 external。
//
// 为什么不是 scope 锁:stop/start 的是同一批容器,两个 scope 各持一个 scope 锁
// 时互相看不见,会同时停/起同一实例——那是假互斥,比没有锁更危险。
type LibraryInstanceMaintenance struct {
Instance string `json:"instance"`
OwnerJobID string `json:"owner_job_id"`
OwnerScope string `json:"owner_scope"`
TargetFingerprint string `json:"target_fingerprint"`
Level string `json:"level"`
Services []string `json:"services"`
IngressState string `json:"ingress_state"`
State string `json:"state"`
OriginalServiceState string `json:"original_service_state"`
Note string `json:"note"`
AcquiredAt time.Time `json:"acquired_at"`
UpdatedAt time.Time `json:"updated_at"`
ReleasedAt *time.Time `json:"released_at,omitempty"`
}
const instanceMaintenanceColumns = `instance, owner_job_id, owner_scope, target_fingerprint, level, services,
ingress_state, state, original_service_state, note, acquired_at, updated_at, released_at`
// AcquireInstanceMaintenance 原子地拿到实例维护权:
// 已释放的记录可以被新作业接管;仍被持有的只允许原作业重入。
func (s *Store) AcquireInstanceMaintenance(ctx context.Context, item LibraryInstanceMaintenance, now time.Time) (LibraryInstanceMaintenance, error) {
level := item.Level
if level == "" {
level = "instance"
}
ingress := item.IngressState
if ingress == "" {
ingress = "unknown"
}
services := item.Services
if services == nil {
services = []string{}
}
raw, err := json.Marshal(services)
if err != nil {
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
}
row := s.db.QueryRow(ctx, `
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level, services,
ingress_state, state, original_service_state, note, acquired_at, updated_at)
VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$11)
ON CONFLICT (instance) DO UPDATE SET
owner_job_id = EXCLUDED.owner_job_id,
owner_scope = EXCLUDED.owner_scope,
target_fingerprint = EXCLUDED.target_fingerprint,
level = EXCLUDED.level,
services = EXCLUDED.services,
ingress_state = EXCLUDED.ingress_state,
state = EXCLUDED.state,
-- 同一作业重入(崩溃接管)时不许用"现在已停"覆盖第一次记下的原状态:
-- 重入时服务已经被我们停住,再读一次只会读到 stopped,写回去会让恢复阶段
-- 以为写入者本来就没跑,于是永远不启动它。
original_service_state = CASE
WHEN library_instance_maintenance.owner_job_id = EXCLUDED.owner_job_id
AND library_instance_maintenance.original_service_state <> ''
THEN library_instance_maintenance.original_service_state
ELSE EXCLUDED.original_service_state
END,
note = EXCLUDED.note,
updated_at = EXCLUDED.updated_at,
released_at = NULL
WHERE library_instance_maintenance.released_at IS NOT NULL
OR library_instance_maintenance.owner_job_id = EXCLUDED.owner_job_id
RETURNING `+instanceMaintenanceColumns,
item.Instance, item.OwnerJobID, item.OwnerScope, item.TargetFingerprint, level, string(raw),
ingress, item.State, item.OriginalServiceState, item.Note, now.UTC())
acquired, err := scanInstanceMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return LibraryInstanceMaintenance{}, ErrMaintenanceHeld
}
if err != nil {
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
}
return acquired, nil
}
// UpdateInstanceMaintenanceState 只允许持有者改自己的窗口状态。
func (s *Store) UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state=$3, note=$4, updated_at=$5
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
if err != nil {
return fmt.Errorf("update instance maintenance: %w", err)
}
if tag.RowsAffected() == 0 {
if s.releasedByThisJob(ctx, "library_instance_maintenance", "instance", instance, ownerJobID) {
return ErrMaintenanceReleased
}
return errors.New("instance maintenance not held by this job")
}
return nil
}
// SetInstanceMaintenanceIngress 记录入口真实状态。
func (s *Store) SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
if state != "closed" && state != "open" {
return errors.New("invalid ingress state")
}
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET ingress_state=$3, note=$4, updated_at=$5
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
if err != nil {
return fmt.Errorf("update instance maintenance ingress: %w", err)
}
if tag.RowsAffected() == 0 {
if s.releasedByThisJob(ctx, "library_instance_maintenance", "instance", instance, ownerJobID) {
return ErrMaintenanceReleased
}
return errors.New("instance maintenance not held by this job")
}
return nil
}
// ReleaseInstanceMaintenance 只有"内容已验证 + 服务与入口恢复"才允许调用。
func (s *Store) ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state='released', released_at=$3, updated_at=$3
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, now.UTC())
if err != nil {
return fmt.Errorf("release instance maintenance: %w", err)
}
if tag.RowsAffected() == 0 {
if s.releasedByThisJob(ctx, "library_instance_maintenance", "instance", instance, ownerJobID) {
return ErrMaintenanceReleased
}
return errors.New("instance maintenance not held by this job")
}
return nil
}
// InstanceMaintenanceOf 读当前实例窗口(接口互斥判断与页面展示都用它)。
func (s *Store) InstanceMaintenanceOf(ctx context.Context, instance string) (*LibraryInstanceMaintenance, error) {
row := s.db.QueryRow(ctx, `SELECT `+instanceMaintenanceColumns+` FROM library_instance_maintenance WHERE instance=$1`, instance)
item, err := scanInstanceMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
// HeldInstanceMaintenance 返回仍然持有(未释放)的实例窗口,供接口在不知道
// 具体实例名时也拒绝写入。
func (s *Store) HeldInstanceMaintenance(ctx context.Context) ([]LibraryInstanceMaintenance, error) {
rows, err := s.db.Query(ctx, `SELECT `+instanceMaintenanceColumns+`
FROM library_instance_maintenance WHERE released_at IS NULL ORDER BY instance`)
if err != nil {
return nil, fmt.Errorf("list instance maintenance: %w", err)
}
defer rows.Close()
items := make([]LibraryInstanceMaintenance, 0, 2)
for rows.Next() {
item, err := scanInstanceMaintenance(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func scanInstanceMaintenance(row pgx.Row) (LibraryInstanceMaintenance, error) {
var item LibraryInstanceMaintenance
var raw []byte
err := row.Scan(&item.Instance, &item.OwnerJobID, &item.OwnerScope, &item.TargetFingerprint, &item.Level,
&raw, &item.IngressState, &item.State, &item.OriginalServiceState, &item.Note,
&item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt)
if err != nil {
return LibraryInstanceMaintenance{}, err
}
if len(raw) > 0 {
if err := json.Unmarshal(raw, &item.Services); err != nil {
return LibraryInstanceMaintenance{}, fmt.Errorf("instance maintenance services unreadable: %w", err)
}
}
return item, nil
}