98294b9583
build / build (push) Failing after 24s
why:审计 19 项(12 P1 / 7 P2)来自同一批根因,逐点试错会把协议撕成十几份:
· 意图与执行混为一谈(F01/F15):`library.sync:<scope>:<快照>` 是终身幂等键,
A→B→A 的第三次发布拿到第一次的历史成功作业,什么都不做;
· 版本行既是实体又是事件(F02/F09):主键只由 (kind,scope,快照) 决定、写入是
upsert,第二个作业覆盖 `source_job_id`/`evidence`/`created_at`;
· 接管没有代次(F13):两个 worker 都能认领同一 pending 作业,失租的旧执行还能
写业务状态、释放别人的维护锁;
· 早退与终态没有统一政策(F03/F04/F05/F06):半写窗口的前置检查失败会重新开门、
服务起不来也照样开入口、journal 终态后收尾不可重建、恢复成功不回写 current;
· 数据身份与准入各写一套(F07/F08/F14/F17):接管丢了复用语义、全库搬运不改
location_hash、通用入口可造 library.* 作业、目标模型从没被问过;
· 存储/容量/UI/CI 判据各自为政(F10/F11/F12/F16/F18/F19)。
what:按批准的一套协议集中实施(详见 handoffs/release-consolidated-implementation/
current-contract.md):
· operation(一次明确意图)/ attempt(epoch 代次)/ 不可变版本事件 / current 指针;
治理库迁移 76,只动治理库,不碰 WeKnora Schema/源码/镜像;
· 所有业务状态、阶段、版本事件、维护锁写入带 epoch 守卫(失租代次写不进去);
心跳失租立刻取消 handler;
· 半写窗口早退保持关闭;服务未全部健康绝不开入口;journal 终态可从持久事实重建
收尾;恢复成功登记 recovery 事件并推进 current;
· worker 在动维护前核对业务行/kind/scope/队列关联;通用入口拒绝 library.*;
· 全库搬运重算 location_hash 并与文档级导入共用实现;接管消费写集合的复用决定;
发布前核目标实时模型契约、容量门禁前置;
· 存储保护闭包改为全量依赖、类型只认显式登记、保护未知则不出可删候选;
· 页面核 HTTP 状态与结构、恢复纳入轮询、Wiki 差异覆盖标题/摘要/父级。
how:`make verify` rc=0(16 项必需检查全绿,含 go test 全量与前端 485 例);
`go vet ./...` rc=0;`npx tsc -p tsconfig.json --noEmit` rc=0;逐项负例见各包测试
(F03/F04/F08/F12/F13/F14/F16/F18/F19 都有先红后绿记录)。
未执行:治理库迁移到任何已有库、真实 PG/ES/S3/WeKnora 集成、故障注入、部署。
656 lines
24 KiB
Go
656 lines
24 KiB
Go
// Package libraryfreeze coordinates a test-only maintenance window. It never
|
||
// grants production authorization or starts services which were already stopped.
|
||
package libraryfreeze
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/rand"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/url"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"syscall"
|
||
"time"
|
||
)
|
||
|
||
type Service struct {
|
||
ID string `json:"id"`
|
||
Running bool `json:"running"`
|
||
Healthy bool `json:"healthy"`
|
||
// ExitCode 是"停下之后"的进程退出码,只有控制器真的读得到时才填。
|
||
// 它是在途请求是否被排空的证据:0 = 服务自己收尾退出,143/137 = 被切断。
|
||
ExitCode *int `json:"exit_code,omitempty"`
|
||
}
|
||
|
||
// GateEvidence 是"入口真的关了、在途真的排空了"的证据。
|
||
// 只有被代理的服务自己不会优雅退出时,才知道为什么要在入口这一层排空。
|
||
type GateEvidence struct {
|
||
Closed bool `json:"closed"`
|
||
Drained bool `json:"drained"`
|
||
ActiveAtClose int `json:"active_at_close"`
|
||
Served int `json:"served"`
|
||
Detail string `json:"detail,omitempty"`
|
||
}
|
||
|
||
// Gate 是目标的唯一对外入口:维护窗口先关它、等在途请求结束,再停写入者。
|
||
type Gate interface {
|
||
// CloseIngress 必须等到在途请求结束才返回成功;返回的不是"已发信号"而是"已排空"。
|
||
CloseIngress(context.Context) (GateEvidence, error)
|
||
// OpenIngress 恢复入口,并确认它真的在服务。
|
||
OpenIngress(context.Context) error
|
||
}
|
||
|
||
// Quiescer 回答"后台写入者是不是已经空了"。
|
||
//
|
||
// 为什么 HTTP 排空不够:被代理的 WeKnora 在 SIGTERM 后会 exit(1),app 内的 asynq
|
||
// worker 正在跑的后台任务会被直接切断。要么等它们跑完,要么明确不进入窗口。
|
||
type Quiescer interface {
|
||
// Idle 返回 true 表示此刻没有排队/在跑的后台任务,也没有未结清的处理记录。
|
||
Idle(ctx context.Context) (bool, string, error)
|
||
}
|
||
|
||
// QuiesceSnapshot 是一次后台任务取样:队列深度 + 任务集合指纹。
|
||
// 指纹的作用是把"这一瞬间恰好是 0"和"一段时间内都没有新任务产生"区分开:
|
||
// scheduled/retry 会自己冒出新任务,只看瞬时 0 无法证明切点。
|
||
type QuiesceSnapshot struct {
|
||
Depth int64 `json:"depth"`
|
||
PendingOps int64 `json:"pending_ops"`
|
||
Unsettled int64 `json:"unsettled"`
|
||
Digest string `json:"digest"`
|
||
Detail string `json:"detail"`
|
||
}
|
||
|
||
// StableQuiescer 能给出任务集合指纹,才谈得上"证明切点"。
|
||
// 实例级维护必须用它;只会回答 Idle 的检查器只能证明某一瞬间是 0。
|
||
type StableQuiescer interface {
|
||
Quiescer
|
||
Snapshot(ctx context.Context) (QuiesceSnapshot, error)
|
||
}
|
||
|
||
// QuiesceProof 是"入口关掉之后后台真的一直是空的"的可复核证据:
|
||
//
|
||
// AfterIngress 之后才开始取样(排空期间在途请求还会派生新任务);
|
||
// 连续 Samples 次取样都为空,且跨过 StableMS 的时间窗;
|
||
// 停写入者之后再取一次样,Depth 仍为 0、指纹不变(否则就是切点不一致=有任务被截断)。
|
||
type QuiesceProof struct {
|
||
AfterIngress time.Time `json:"after_ingress"`
|
||
Samples int `json:"samples"`
|
||
StableMS int64 `json:"stable_ms"`
|
||
Depth int64 `json:"depth"`
|
||
PendingOps int64 `json:"pending_ops"`
|
||
Unsettled int64 `json:"unsettled"`
|
||
Digest string `json:"digest"`
|
||
Detail string `json:"detail"`
|
||
ProvenAt time.Time `json:"proven_at"`
|
||
PostStopDepth int64 `json:"post_stop_depth"`
|
||
PostStopDigest string `json:"post_stop_digest"`
|
||
PostStopAt time.Time `json:"post_stop_at"`
|
||
}
|
||
|
||
// Inventory must enumerate the complete, independently scoped test writer set,
|
||
// using immutable instance IDs. Never implement it as only the currently running
|
||
// subset. Start must wait for readiness; callers still read back the result.
|
||
type Controller interface {
|
||
Inventory(context.Context) ([]Service, error)
|
||
Stop(context.Context, string) error
|
||
Start(context.Context, string) error
|
||
}
|
||
type Result struct {
|
||
State string `json:"state"`
|
||
SnapshotID string `json:"snapshot_id,omitempty"`
|
||
// Digest 是"这次完成的校验摘要"(恢复路径用它证明内容身份)。
|
||
// 终态重放时从 journal 里读回,调用方据此补做业务收尾而不重跑数据。
|
||
Digest string `json:"digest,omitempty"`
|
||
// Replayed 标记"这次没有执行数据动作,只是把已完成的终态读回来"。
|
||
Replayed bool `json:"replayed,omitempty"`
|
||
}
|
||
type serviceRecord struct {
|
||
Service
|
||
StopRequested bool `json:"stop_requested"`
|
||
StopVerified bool `json:"stop_verified"`
|
||
Restored bool `json:"restored"`
|
||
}
|
||
type journal struct {
|
||
Version int `json:"version"`
|
||
Source string `json:"source"`
|
||
Job string `json:"job"`
|
||
Result
|
||
// Level 区分"实例级维护"(写入者/入口被真的关掉)与"仅治理库互斥"。
|
||
// 没有 Level 的旧 journal 来自封存链路,按实例级处理但不冒充已验证。
|
||
Level string `json:"level,omitempty"`
|
||
Ingress string `json:"ingress,omitempty"`
|
||
IngressClosed bool `json:"ingress_closed,omitempty"`
|
||
DrainVerified bool `json:"drain_verified,omitempty"`
|
||
ContentRisk bool `json:"content_risk,omitempty"`
|
||
RestoreBlocked bool `json:"restore_blocked,omitempty"`
|
||
// Gate 记录入口关闭/排空的证据;没有它就不能说"入口关了"。
|
||
// 入口恢复后证据仍保留(GateReopened=true),否则事后查不到
|
||
// "关的时候还有几个在途请求"。
|
||
Gate *GateEvidence `json:"gate,omitempty"`
|
||
GateReopened bool `json:"gate_reopened,omitempty"`
|
||
// BackupSnapshotID 是这次窗口开始写之前落盘的那份预备份:恢复只能按它来,
|
||
// 不允许事后换一份"看起来更好"的快照。
|
||
BackupSnapshotID string `json:"backup_snapshot_id,omitempty"`
|
||
// TargetFingerprint 是开窗口时的目标身份:恢复时必须还是同一个目标。
|
||
TargetFingerprint string `json:"target_fingerprint,omitempty"`
|
||
// BackgroundIdle 只记录"这个窗口曾经等到过空队列";它是历史事实,
|
||
// 不能当作"再次开入口/重启之后仍然静默"的证明——那种判断只认 Quiesce。
|
||
BackgroundIdle bool `json:"background_idle,omitempty"`
|
||
// Quiesce 是入口关闭之后的后台切点证据(稳定窗口 + 停写者后的复核)。
|
||
Quiesce *QuiesceProof `json:"quiesce,omitempty"`
|
||
// 恢复证据:只有真正跑完"恢复 + 校验"才会写这三项。
|
||
RecoveredBackup string `json:"recovered_backup,omitempty"`
|
||
RecoveryDigest string `json:"recovery_digest,omitempty"`
|
||
RecoveredAt *time.Time `json:"recovered_at,omitempty"`
|
||
Services []serviceRecord `json:"services"`
|
||
StartedAt time.Time `json:"started_at"`
|
||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||
Code string `json:"code,omitempty"`
|
||
}
|
||
type Coordinator struct {
|
||
root *os.Root
|
||
source string
|
||
control Controller
|
||
level string
|
||
ingress string
|
||
// ingressClosed / ingressReady 是可选的外部探针:声明了实例级维护就必须
|
||
// 证明入口真的关了、真的回来了,而不是"停了容器就算关"。
|
||
ingressClosed func(context.Context) error
|
||
ingressReady func(context.Context) error
|
||
gate Gate
|
||
quiescer Quiescer
|
||
fingerprint string
|
||
quiesceWait time.Duration
|
||
// quiesceStable 是"连续为空必须跨过的时间窗":单次瞬时 0 不算证明。
|
||
quiesceStable time.Duration
|
||
// quiesceStall 是"同一批任务多久没有进展就判定卡住/被截断"。
|
||
quiesceStall time.Duration
|
||
}
|
||
|
||
func Open(root, source string, c Controller) (*Coordinator, error) {
|
||
if !filepath.IsAbs(root) || !validID(source) || c == nil {
|
||
return nil, errors.New("invalid freeze configuration")
|
||
}
|
||
info, err := os.Lstat(root)
|
||
if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 {
|
||
return nil, errors.New("freeze directory must be private and real")
|
||
}
|
||
resolved, err := filepath.EvalSymlinks(root)
|
||
if err != nil || resolved != filepath.Clean(root) {
|
||
return nil, errors.New("freeze directory contains symlink")
|
||
}
|
||
r, err := os.OpenRoot(root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return &Coordinator{root: r, source: source, control: c}, nil
|
||
}
|
||
func (c *Coordinator) Close() error { return c.root.Close() }
|
||
|
||
// SetTarget 声明这次维护的层级与入口。实例级维护必须指名入口地址;
|
||
// 只锁治理 scope 的维护不许带入口,否则就是拿"库级"冒充"实例级"。
|
||
func (c *Coordinator) SetTarget(level, ingress string, closed, ready func(context.Context) error) error {
|
||
switch level {
|
||
case "instance":
|
||
if !validIngress(ingress) {
|
||
return errors.New("instance maintenance requires the target ingress address")
|
||
}
|
||
case "library":
|
||
if ingress != "" || closed != nil || ready != nil {
|
||
return errors.New("library scope lock must not claim ingress control")
|
||
}
|
||
default:
|
||
return errors.New("invalid maintenance level")
|
||
}
|
||
c.level, c.ingress, c.ingressClosed, c.ingressReady = level, ingress, closed, ready
|
||
return nil
|
||
}
|
||
|
||
// SetGate 声明目标入口闸门:实例级维护里"关入口"就是关它、并从它拿到排空证据。
|
||
func (c *Coordinator) SetGate(g Gate) error {
|
||
if g == nil {
|
||
return errors.New("invalid ingress gate")
|
||
}
|
||
c.gate = g
|
||
return nil
|
||
}
|
||
|
||
// SetTargetFingerprint 绑住"这次维护动的是哪个目标"。恢复时必须与 journal
|
||
// 里记录的一致,否则说明配置换了目标,禁止拿旧窗口去收尾新目标。
|
||
func (c *Coordinator) SetTargetFingerprint(fingerprint string) error {
|
||
if len(fingerprint) != 64 {
|
||
return errors.New("invalid target fingerprint")
|
||
}
|
||
if _, err := hex.DecodeString(fingerprint); err != nil {
|
||
return errors.New("invalid target fingerprint")
|
||
}
|
||
c.fingerprint = strings.ToLower(fingerprint)
|
||
return nil
|
||
}
|
||
|
||
// SetQuiesceTimeout 调整"等后台任务静默"的上限(默认 2 分钟)。
|
||
func (c *Coordinator) SetQuiesceTimeout(d time.Duration) error {
|
||
if d <= 0 {
|
||
return errors.New("invalid quiesce timeout")
|
||
}
|
||
c.quiesceWait = d
|
||
return nil
|
||
}
|
||
|
||
// SetQuiesceStability 调整"稳定切点"的时间窗与任务停滞判定(默认 1.5s / 30s)。
|
||
func (c *Coordinator) SetQuiesceStability(stable, stall time.Duration) error {
|
||
if stable <= 0 || stall <= 0 {
|
||
return errors.New("invalid quiesce stability window")
|
||
}
|
||
c.quiesceStable, c.quiesceStall = stable, stall
|
||
return nil
|
||
}
|
||
|
||
// SetQuiescer 声明后台任务静默检查(HTTP 排空不等于后台任务排空)。
|
||
func (c *Coordinator) SetQuiescer(q Quiescer) error {
|
||
if q == nil {
|
||
return errors.New("invalid quiescer")
|
||
}
|
||
c.quiescer = q
|
||
return nil
|
||
}
|
||
|
||
func validIngress(raw string) bool {
|
||
parsed, err := url.Parse(raw)
|
||
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.Path != "" && parsed.User == nil
|
||
}
|
||
|
||
func (c *Coordinator) Recover(ctx context.Context) error {
|
||
lock, err := c.lock()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer unlock(lock)
|
||
dir, err := c.root.Open(".")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer dir.Close()
|
||
entries, err := dir.ReadDir(-1)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
// Validate every journal before issuing recovery writes. Corrupt records or
|
||
// another source's directory must not turn into broad service startup.
|
||
var pending []journal
|
||
for _, entry := range entries {
|
||
if !strings.HasSuffix(entry.Name(), ".json") {
|
||
continue
|
||
}
|
||
job := strings.TrimSuffix(entry.Name(), ".json")
|
||
if !validID(job) {
|
||
return errors.New("invalid freeze journal name")
|
||
}
|
||
j, err := c.load(job)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !terminal(j.State) {
|
||
pending = append(pending, j)
|
||
}
|
||
}
|
||
// 内容可能已被写过的窗口不能自动开放:重启恢复只许"收尾确定性的事",
|
||
// 半写目标必须由原作业接管或人工按预备份恢复,否则就是把半写当成品放行。
|
||
for _, j := range pending {
|
||
if j.ContentRisk || j.RestoreBlocked {
|
||
return fmt.Errorf("维护窗口 %s 的内容状态未验证(可能已半写):目标保持停用,须由原作业接管或按预备份恢复", j.Job)
|
||
}
|
||
}
|
||
if len(pending) > 1 {
|
||
return errors.New("multiple interrupted freezes require operator reconciliation")
|
||
}
|
||
for _, j := range pending {
|
||
if err = c.restore(ctx, &j); err != nil {
|
||
return err
|
||
}
|
||
j.State = "failed"
|
||
j.Code = "interrupted"
|
||
ended := time.Now().UTC()
|
||
j.FinishedAt = &ended
|
||
if err = c.save(j); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Inspect reports the persisted result of a job, if this coordinator ever ran it.
|
||
// A caller which re-uses a job ID needs the original request identity (time,
|
||
// scope, knowledge base, contract) to replay idempotently instead of guessing a
|
||
// fresh one, which would be rejected as a conflicting request.
|
||
func (c *Coordinator) Inspect(job string) (Result, bool, error) {
|
||
if !validID(job) {
|
||
return Result{}, false, errors.New("invalid freeze job id")
|
||
}
|
||
j, err := c.load(job)
|
||
if errors.Is(err, os.ErrNotExist) {
|
||
return Result{}, false, nil
|
||
}
|
||
if err != nil {
|
||
return Result{}, false, err
|
||
}
|
||
return j.Result, true, nil
|
||
}
|
||
|
||
func invokeCapture(ctx context.Context, capture func(context.Context) (string, error)) (id string, err error) {
|
||
defer func() {
|
||
if recover() != nil {
|
||
id = ""
|
||
err = errors.New("candidate capture panicked")
|
||
}
|
||
}()
|
||
return capture(ctx)
|
||
}
|
||
|
||
func validID(s string) bool {
|
||
if len(s) == 0 || len(s) > 128 {
|
||
return false
|
||
}
|
||
for _, r := range s {
|
||
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_') {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
func terminal(s string) bool { return s == "succeeded" || s == "failed" || s == "recovered" }
|
||
|
||
// A kernel lock serializes the whole test writer group, not individual KBs.
|
||
// A crash releases the lock, but leaves the write-ahead journal for recovery.
|
||
func (c *Coordinator) lock() (*os.File, error) {
|
||
f, err := c.root.OpenFile("operation.lock", os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0600)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||
f.Close()
|
||
return nil, errors.New("test source freeze is busy")
|
||
}
|
||
return f, nil
|
||
}
|
||
func unlock(f *os.File) { _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN); _ = f.Close() }
|
||
func (c *Coordinator) save(j journal) error {
|
||
data, err := json.Marshal(j)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
var token [16]byte
|
||
if _, err = rand.Read(token[:]); err != nil {
|
||
return err
|
||
}
|
||
name := "write-" + hex.EncodeToString(token[:]) + ".tmp"
|
||
f, err := c.root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer c.root.Remove(name)
|
||
defer f.Close()
|
||
if _, err = f.Write(data); err != nil {
|
||
return err
|
||
}
|
||
if err = f.Sync(); err != nil {
|
||
return err
|
||
}
|
||
if err = f.Close(); err != nil {
|
||
return err
|
||
}
|
||
if err = c.root.Rename(name, j.Job+".json"); err != nil {
|
||
return err
|
||
}
|
||
dir, err := c.root.Open(".")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer dir.Close()
|
||
return dir.Sync()
|
||
}
|
||
func (c *Coordinator) load(job string) (journal, error) {
|
||
f, err := c.root.OpenFile(job+".json", os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
||
if err != nil {
|
||
return journal{}, err
|
||
}
|
||
defer f.Close()
|
||
info, err := f.Stat()
|
||
if err != nil || !info.Mode().IsRegular() || info.Size() > 1024*1024 {
|
||
return journal{}, errors.New("invalid freeze journal file")
|
||
}
|
||
data, err := io.ReadAll(io.LimitReader(f, 1024*1024+1))
|
||
if err != nil {
|
||
return journal{}, err
|
||
}
|
||
var j journal
|
||
dec := json.NewDecoder(bytes.NewReader(data))
|
||
dec.DisallowUnknownFields()
|
||
if err = dec.Decode(&j); err != nil {
|
||
return journal{}, errors.New("invalid freeze journal")
|
||
}
|
||
var extra any
|
||
if dec.Decode(&extra) != io.EOF || j.Version != 1 || j.Source != c.source || j.Job != job || !validID(j.Job) {
|
||
return journal{}, errors.New("freeze journal identity mismatch")
|
||
}
|
||
switch j.State {
|
||
case "stopping", "held", "capturing", "recovering", "restoring", "succeeded", "failed", "recovered":
|
||
default:
|
||
return journal{}, errors.New("invalid freeze journal state")
|
||
}
|
||
if j.Level != "" && j.Level != "instance" && j.Level != "library" {
|
||
return journal{}, errors.New("invalid freeze maintenance level")
|
||
}
|
||
// 声明了实例级维护就必须留下入口证据;只锁库的窗口不许带入口。
|
||
if j.Level == "instance" && !validIngress(j.Ingress) {
|
||
return journal{}, errors.New("invalid freeze ingress evidence")
|
||
}
|
||
if j.Level == "library" && (j.Ingress != "" || j.IngressClosed) {
|
||
return journal{}, errors.New("library scope lock claims ingress control")
|
||
}
|
||
if j.RestoreBlocked && !j.ContentRisk {
|
||
return journal{}, errors.New("invalid freeze restore block")
|
||
}
|
||
if j.Gate != nil && (!j.Gate.Closed || j.Gate.ActiveAtClose < 0 || j.Gate.Served < 0) {
|
||
return journal{}, errors.New("invalid freeze ingress evidence")
|
||
}
|
||
if j.GateReopened && j.Gate == nil {
|
||
return journal{}, errors.New("ingress reopened without close evidence")
|
||
}
|
||
if j.BackupSnapshotID != "" && !validDigest(j.BackupSnapshotID) {
|
||
return journal{}, errors.New("invalid backup snapshot id")
|
||
}
|
||
// 后台静默只认带稳定窗口与停写后复核的证据。旧的 background_idle 布尔
|
||
// 只是历史记录:它不能证明"重启/再开入口之后仍然静默",也不能证明停写者
|
||
// 那一刻没有任务在跑。这里允许它存在(生存旧 journal),但绝不能当证明用:
|
||
// RecoverWindow 每次都会重新取样,不读这个布尔。
|
||
if j.BackgroundIdle && j.Quiesce == nil && !terminal(j.State) {
|
||
// 非终态窗口没有证明:状态上就是"还没证明静默",交给恢复流程重新取证。
|
||
j.BackgroundIdle = false
|
||
}
|
||
if q := j.Quiesce; q != nil {
|
||
if q.Samples < 2 || q.StableMS < 0 || q.Depth != 0 || q.PendingOps != 0 || q.Unsettled != 0 {
|
||
return journal{}, errors.New("invalid quiesce proof")
|
||
}
|
||
if !validDigest(q.Digest) || q.AfterIngress.IsZero() || q.ProvenAt.IsZero() || q.ProvenAt.Before(q.AfterIngress) {
|
||
return journal{}, errors.New("invalid quiesce proof")
|
||
}
|
||
if !q.PostStopAt.IsZero() && (q.PostStopDepth != 0 || q.PostStopDigest != q.Digest || q.PostStopAt.Before(q.ProvenAt)) {
|
||
return journal{}, errors.New("invalid post-stop quiesce proof")
|
||
}
|
||
}
|
||
// 恢复证据只允许指向"这次窗口自己那份预备份",且必须带校验摘要。
|
||
if j.RecoveredAt != nil {
|
||
if j.RecoveredBackup == "" || j.RecoveredBackup != j.BackupSnapshotID || j.RecoveryDigest == "" {
|
||
return journal{}, errors.New("invalid recovery evidence")
|
||
}
|
||
}
|
||
if len(j.Services) == 0 || len(j.Services) > 256 {
|
||
return journal{}, errors.New("invalid freeze journal services")
|
||
}
|
||
for i, s := range j.Services {
|
||
if !validID(s.ID) || (i > 0 && j.Services[i-1].ID >= s.ID) || s.StopRequested && !s.Running || s.Restored && (!s.StopRequested || !s.Running) {
|
||
return journal{}, errors.New("invalid freeze service ownership")
|
||
}
|
||
// 退出码只能出现在"被要求停过"的服务上;Running 记的是原始状态。
|
||
if s.ExitCode != nil && !s.StopRequested {
|
||
return journal{}, errors.New("invalid freeze drain evidence")
|
||
}
|
||
}
|
||
return j, nil
|
||
}
|
||
func (c *Coordinator) pending() error {
|
||
dir, err := c.root.Open(".")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer dir.Close()
|
||
entries, err := dir.ReadDir(-1)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, entry := range entries {
|
||
if strings.HasSuffix(entry.Name(), ".json") {
|
||
job := strings.TrimSuffix(entry.Name(), ".json")
|
||
if !validID(job) {
|
||
return errors.New("invalid freeze journal name")
|
||
}
|
||
j, err := c.load(job)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !terminal(j.State) {
|
||
return errors.New("interrupted test freeze requires recovery")
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
func inventory(ctx context.Context, control Controller) ([]Service, error) {
|
||
items, err := control.Inventory(ctx)
|
||
if err != nil {
|
||
return nil, errors.New("test writer inventory unavailable")
|
||
}
|
||
if len(items) == 0 || len(items) > 256 {
|
||
return nil, errors.New("invalid test writer inventory")
|
||
}
|
||
seen := map[string]bool{}
|
||
for _, s := range items {
|
||
if !validID(s.ID) || seen[s.ID] {
|
||
return nil, errors.New("invalid test writer identity")
|
||
}
|
||
seen[s.ID] = true
|
||
}
|
||
sort.Slice(items, func(i, j int) bool { return items[i].ID < items[j].ID })
|
||
return items, nil
|
||
}
|
||
func (c *Coordinator) stopped(ctx context.Context, j journal) error {
|
||
items, err := inventory(ctx, c.control)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if len(items) != len(j.Services) {
|
||
return errors.New("test writer topology changed")
|
||
}
|
||
for i, s := range items {
|
||
if s.ID != j.Services[i].ID || s.Running {
|
||
return errors.New("test writers not isolated")
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
func (c *Coordinator) restore(ctx context.Context, j *journal) error {
|
||
j.State = "restoring"
|
||
// 两类失败必须分开累计:
|
||
// persistFailures:journal 落盘失败。它让证据不完整,但不改变"服务是否真的回来了"。
|
||
// serviceFailures:启动/健康/拓扑不确定。只要有一项,就绝不能开入口——
|
||
// 状态字段写 held 而外部入口已开,是对操作者和流量的双重谎报(F04)。
|
||
var persistFailures error = c.save(*j)
|
||
var serviceFailures error
|
||
for i := range j.Services {
|
||
s := &j.Services[i]
|
||
if !s.Running || !s.StopRequested {
|
||
continue
|
||
}
|
||
items, err := inventory(ctx, c.control)
|
||
if err != nil {
|
||
serviceFailures = errors.Join(serviceFailures, err)
|
||
continue
|
||
}
|
||
found := false
|
||
for _, actual := range items {
|
||
if actual.ID != s.ID {
|
||
continue
|
||
}
|
||
found = true
|
||
if actual.Running && !s.StopVerified {
|
||
serviceFailures = errors.Join(serviceFailures, errors.New("stop outcome remains uncertain; service may still stop later"))
|
||
break
|
||
}
|
||
if !actual.Running {
|
||
s.StopVerified = true
|
||
if saveErr := c.save(*j); saveErr != nil {
|
||
persistFailures = errors.Join(persistFailures, saveErr)
|
||
}
|
||
if err = c.control.Start(ctx, s.ID); err != nil {
|
||
serviceFailures = errors.Join(serviceFailures, errors.New("test service restart failed"))
|
||
break
|
||
}
|
||
}
|
||
after, readErr := inventory(ctx, c.control)
|
||
healthy := false
|
||
if readErr == nil {
|
||
for _, a := range after {
|
||
if a.ID == s.ID && a.Running && a.Healthy {
|
||
healthy = true
|
||
}
|
||
}
|
||
}
|
||
if !healthy {
|
||
serviceFailures = errors.Join(serviceFailures, errors.New("test service health unverified"))
|
||
break
|
||
}
|
||
s.Restored = true
|
||
if err = c.save(*j); err != nil {
|
||
persistFailures = errors.Join(persistFailures, err)
|
||
}
|
||
break
|
||
}
|
||
if !found {
|
||
serviceFailures = errors.Join(serviceFailures, errors.New("original test service missing; replacement not started"))
|
||
}
|
||
}
|
||
// 服务都回来了才开放入口:先开门再启动服务,等于把流量打向没准备好的目标。
|
||
//
|
||
// 注意:入口关到一半失败(闸门已被停、排空超时)时 j.Gate 可能还是 nil,
|
||
// 但入口确实已经不在了。所以实例级维护一律重新打开入口,而不是只看
|
||
// "有没有成功关闭的证据"——否则会出现"维护已释放、服务在跑、入口却是关的"。
|
||
// 但只要**任何一个写入者没回到健康**,这里的 else 分支就不执行:入口保持关闭,
|
||
// 由调用方按 held 处理并走恢复(宁可不可用,也不对外暴露没准备好的目标)。
|
||
switch {
|
||
case serviceFailures != nil:
|
||
serviceFailures = errors.Join(serviceFailures, errors.New("目标入口保持关闭:写入者未全部恢复健康"))
|
||
case c.level == "instance" || j.Gate != nil:
|
||
if c.gate == nil {
|
||
persistFailures = errors.Join(persistFailures, errors.New("入口闸门缺失:不能确认目标已恢复"))
|
||
} else if err := c.gate.OpenIngress(ctx); err != nil {
|
||
persistFailures = errors.Join(persistFailures, fmt.Errorf("目标入口恢复失败:%w", err))
|
||
} else if j.Gate != nil {
|
||
// 只有"确实关过"的窗口才记"重新打开":没有关闭证据时开门是恢复原状,
|
||
// 不能写成"重开",否则 journal 校验会误判。
|
||
j.GateReopened = true
|
||
if err := c.save(*j); err != nil {
|
||
persistFailures = errors.Join(persistFailures, err)
|
||
}
|
||
}
|
||
}
|
||
return errors.Join(serviceFailures, persistFailures)
|
||
}
|