c5e033dd0e
build / build (push) Failing after 15s
- 同作业号重跑读持久日志+已封存清单按首次请求重放(source_read=false, writers_stopped=false),换范围/绑定库或清单校验失败即拒绝 - 归档用量按跨版本去重统计内容字节,清单/未引用 blob/实际分配单列, 不再用各版本 unique 之和冒充归档占用 - 冻结窗口内复核来源契约,不一致即中止;拒绝组外同镜像运行容器 - 补齐整库只读接口的 OpenAPI 契约 - 按用户确认取消 A06:文档/页面/接口不再把效果测试列为封版门槛
492 lines
13 KiB
Go
492 lines
13 KiB
Go
// Package libraryfreeze coordinates a test-only maintenance window. It never
|
|
// grants production authorization or starts services which were already stopped.
|
|
package libraryfreeze
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
type Service struct {
|
|
ID string `json:"id"`
|
|
Running bool `json:"running"`
|
|
Healthy bool `json:"healthy"`
|
|
}
|
|
|
|
// Inventory must enumerate the complete, independently scoped test writer set,
|
|
// using immutable instance IDs. Never implement it as only the currently running
|
|
// subset. Start must wait for readiness; callers still read back the result.
|
|
type Controller interface {
|
|
Inventory(context.Context) ([]Service, error)
|
|
Stop(context.Context, string) error
|
|
Start(context.Context, string) error
|
|
}
|
|
type Result struct {
|
|
State string `json:"state"`
|
|
SnapshotID string `json:"snapshot_id,omitempty"`
|
|
}
|
|
type serviceRecord struct {
|
|
Service
|
|
StopRequested bool `json:"stop_requested"`
|
|
StopVerified bool `json:"stop_verified"`
|
|
Restored bool `json:"restored"`
|
|
}
|
|
type journal struct {
|
|
Version int `json:"version"`
|
|
Source string `json:"source"`
|
|
Job string `json:"job"`
|
|
Result
|
|
Services []serviceRecord `json:"services"`
|
|
StartedAt time.Time `json:"started_at"`
|
|
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
|
Code string `json:"code,omitempty"`
|
|
}
|
|
type Coordinator struct {
|
|
root *os.Root
|
|
source string
|
|
control Controller
|
|
}
|
|
|
|
func Open(root, source string, c Controller) (*Coordinator, error) {
|
|
if !filepath.IsAbs(root) || !validID(source) || c == nil {
|
|
return nil, errors.New("invalid freeze configuration")
|
|
}
|
|
info, err := os.Lstat(root)
|
|
if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 {
|
|
return nil, errors.New("freeze directory must be private and real")
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(root)
|
|
if err != nil || resolved != filepath.Clean(root) {
|
|
return nil, errors.New("freeze directory contains symlink")
|
|
}
|
|
r, err := os.OpenRoot(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Coordinator{root: r, source: source, control: c}, nil
|
|
}
|
|
func (c *Coordinator) Close() error { return c.root.Close() }
|
|
|
|
func (c *Coordinator) Recover(ctx context.Context) error {
|
|
lock, err := c.lock()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer unlock(lock)
|
|
dir, err := c.root.Open(".")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer dir.Close()
|
|
entries, err := dir.ReadDir(-1)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Validate every journal before issuing recovery writes. Corrupt records or
|
|
// another source's directory must not turn into broad service startup.
|
|
var pending []journal
|
|
for _, entry := range entries {
|
|
if !strings.HasSuffix(entry.Name(), ".json") {
|
|
continue
|
|
}
|
|
job := strings.TrimSuffix(entry.Name(), ".json")
|
|
if !validID(job) {
|
|
return errors.New("invalid freeze journal name")
|
|
}
|
|
j, err := c.load(job)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !terminal(j.State) {
|
|
pending = append(pending, j)
|
|
}
|
|
}
|
|
if len(pending) > 1 {
|
|
return errors.New("multiple interrupted freezes require operator reconciliation")
|
|
}
|
|
for _, j := range pending {
|
|
if err = c.restore(ctx, &j); err != nil {
|
|
return err
|
|
}
|
|
j.State = "failed"
|
|
j.Code = "interrupted"
|
|
ended := time.Now().UTC()
|
|
j.FinishedAt = &ended
|
|
if err = c.save(j); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Inspect reports the persisted result of a job, if this coordinator ever ran it.
|
|
// A caller which re-uses a job ID needs the original request identity (time,
|
|
// scope, knowledge base, contract) to replay idempotently instead of guessing a
|
|
// fresh one, which would be rejected as a conflicting request.
|
|
func (c *Coordinator) Inspect(job string) (Result, bool, error) {
|
|
if !validID(job) {
|
|
return Result{}, false, errors.New("invalid freeze job id")
|
|
}
|
|
j, err := c.load(job)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return Result{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return Result{}, false, err
|
|
}
|
|
return j.Result, true, nil
|
|
}
|
|
|
|
func invokeCapture(ctx context.Context, capture func(context.Context) (string, error)) (id string, err error) {
|
|
defer func() {
|
|
if recover() != nil {
|
|
id = ""
|
|
err = errors.New("candidate capture panicked")
|
|
}
|
|
}()
|
|
return capture(ctx)
|
|
}
|
|
|
|
func validID(s string) bool {
|
|
if len(s) == 0 || len(s) > 128 {
|
|
return false
|
|
}
|
|
for _, r := range s {
|
|
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
func terminal(s string) bool { return s == "succeeded" || s == "failed" }
|
|
|
|
// A kernel lock serializes the whole test writer group, not individual KBs.
|
|
// A crash releases the lock, but leaves the write-ahead journal for recovery.
|
|
func (c *Coordinator) lock() (*os.File, error) {
|
|
f, err := c.root.OpenFile("operation.lock", os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0600)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
|
f.Close()
|
|
return nil, errors.New("test source freeze is busy")
|
|
}
|
|
return f, nil
|
|
}
|
|
func unlock(f *os.File) { _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN); _ = f.Close() }
|
|
func (c *Coordinator) save(j journal) error {
|
|
data, err := json.Marshal(j)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var token [16]byte
|
|
if _, err = rand.Read(token[:]); err != nil {
|
|
return err
|
|
}
|
|
name := "write-" + hex.EncodeToString(token[:]) + ".tmp"
|
|
f, err := c.root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer c.root.Remove(name)
|
|
defer f.Close()
|
|
if _, err = f.Write(data); err != nil {
|
|
return err
|
|
}
|
|
if err = f.Sync(); err != nil {
|
|
return err
|
|
}
|
|
if err = f.Close(); err != nil {
|
|
return err
|
|
}
|
|
if err = c.root.Rename(name, j.Job+".json"); err != nil {
|
|
return err
|
|
}
|
|
dir, err := c.root.Open(".")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer dir.Close()
|
|
return dir.Sync()
|
|
}
|
|
func (c *Coordinator) load(job string) (journal, error) {
|
|
f, err := c.root.OpenFile(job+".json", os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
return journal{}, err
|
|
}
|
|
defer f.Close()
|
|
info, err := f.Stat()
|
|
if err != nil || !info.Mode().IsRegular() || info.Size() > 1024*1024 {
|
|
return journal{}, errors.New("invalid freeze journal file")
|
|
}
|
|
data, err := io.ReadAll(io.LimitReader(f, 1024*1024+1))
|
|
if err != nil {
|
|
return journal{}, err
|
|
}
|
|
var j journal
|
|
dec := json.NewDecoder(bytes.NewReader(data))
|
|
dec.DisallowUnknownFields()
|
|
if err = dec.Decode(&j); err != nil {
|
|
return journal{}, errors.New("invalid freeze journal")
|
|
}
|
|
var extra any
|
|
if dec.Decode(&extra) != io.EOF || j.Version != 1 || j.Source != c.source || j.Job != job || !validID(j.Job) {
|
|
return journal{}, errors.New("freeze journal identity mismatch")
|
|
}
|
|
switch j.State {
|
|
case "stopping", "capturing", "restoring", "succeeded", "failed":
|
|
default:
|
|
return journal{}, errors.New("invalid freeze journal state")
|
|
}
|
|
if len(j.Services) == 0 || len(j.Services) > 256 {
|
|
return journal{}, errors.New("invalid freeze journal services")
|
|
}
|
|
for i, s := range j.Services {
|
|
if !validID(s.ID) || (i > 0 && j.Services[i-1].ID >= s.ID) || s.StopRequested && !s.Running || s.Restored && (!s.StopRequested || !s.Running) {
|
|
return journal{}, errors.New("invalid freeze service ownership")
|
|
}
|
|
}
|
|
return j, nil
|
|
}
|
|
func (c *Coordinator) pending() error {
|
|
dir, err := c.root.Open(".")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer dir.Close()
|
|
entries, err := dir.ReadDir(-1)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
if strings.HasSuffix(entry.Name(), ".json") {
|
|
job := strings.TrimSuffix(entry.Name(), ".json")
|
|
if !validID(job) {
|
|
return errors.New("invalid freeze journal name")
|
|
}
|
|
j, err := c.load(job)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !terminal(j.State) {
|
|
return errors.New("interrupted test freeze requires recovery")
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func inventory(ctx context.Context, control Controller) ([]Service, error) {
|
|
items, err := control.Inventory(ctx)
|
|
if err != nil {
|
|
return nil, errors.New("test writer inventory unavailable")
|
|
}
|
|
if len(items) == 0 || len(items) > 256 {
|
|
return nil, errors.New("invalid test writer inventory")
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, s := range items {
|
|
if !validID(s.ID) || seen[s.ID] {
|
|
return nil, errors.New("invalid test writer identity")
|
|
}
|
|
seen[s.ID] = true
|
|
}
|
|
sort.Slice(items, func(i, j int) bool { return items[i].ID < items[j].ID })
|
|
return items, nil
|
|
}
|
|
func (c *Coordinator) stopped(ctx context.Context, j journal) error {
|
|
items, err := inventory(ctx, c.control)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(items) != len(j.Services) {
|
|
return errors.New("test writer topology changed")
|
|
}
|
|
for i, s := range items {
|
|
if s.ID != j.Services[i].ID || s.Running {
|
|
return errors.New("test writers not isolated")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func (c *Coordinator) restore(ctx context.Context, j *journal) error {
|
|
j.State = "restoring"
|
|
// Restoring services is still necessary if the journal disk fails after a
|
|
// persisted stop intent. Keep the error, but do not let it suppress recovery.
|
|
var failures error = c.save(*j)
|
|
for i := range j.Services {
|
|
s := &j.Services[i]
|
|
if !s.Running || !s.StopRequested {
|
|
continue
|
|
}
|
|
items, err := inventory(ctx, c.control)
|
|
if err != nil {
|
|
failures = errors.Join(failures, err)
|
|
continue
|
|
}
|
|
found := false
|
|
for _, actual := range items {
|
|
if actual.ID != s.ID {
|
|
continue
|
|
}
|
|
found = true
|
|
if actual.Running && !s.StopVerified {
|
|
failures = errors.Join(failures, errors.New("stop outcome remains uncertain; service may still stop later"))
|
|
break
|
|
}
|
|
if !actual.Running {
|
|
s.StopVerified = true
|
|
if saveErr := c.save(*j); saveErr != nil {
|
|
failures = errors.Join(failures, saveErr)
|
|
}
|
|
if err = c.control.Start(ctx, s.ID); err != nil {
|
|
failures = errors.Join(failures, errors.New("test service restart failed"))
|
|
break
|
|
}
|
|
}
|
|
after, readErr := inventory(ctx, c.control)
|
|
healthy := false
|
|
if readErr == nil {
|
|
for _, a := range after {
|
|
if a.ID == s.ID && a.Running && a.Healthy {
|
|
healthy = true
|
|
}
|
|
}
|
|
}
|
|
if !healthy {
|
|
failures = errors.Join(failures, errors.New("test service health unverified"))
|
|
break
|
|
}
|
|
s.Restored = true
|
|
if err = c.save(*j); err != nil {
|
|
failures = errors.Join(failures, err)
|
|
}
|
|
break
|
|
}
|
|
if !found {
|
|
failures = errors.Join(failures, errors.New("original test service missing; replacement not started"))
|
|
}
|
|
}
|
|
return failures
|
|
}
|
|
func (c *Coordinator) Run(ctx context.Context, job string, capture func(context.Context) (string, error)) (Result, error) {
|
|
return c.RunPrepared(ctx, job, func(context.Context) error { return nil }, capture)
|
|
}
|
|
|
|
// RunPrepared runs readiness checks under the source lock, before any stop
|
|
// intent. It does not repeat checks or capture for an already completed job.
|
|
func (c *Coordinator) RunPrepared(ctx context.Context, job string, prepare func(context.Context) error, capture func(context.Context) (string, error)) (Result, error) {
|
|
if !validID(job) || capture == nil || prepare == nil {
|
|
return Result{}, errors.New("invalid freeze request")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return Result{}, err
|
|
}
|
|
lock, err := c.lock()
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
defer unlock(lock)
|
|
previous, err := c.load(job)
|
|
if err == nil {
|
|
if previous.State == "succeeded" {
|
|
return previous.Result, nil
|
|
}
|
|
return previous.Result, errors.New("freeze request already exists; recovery or a new request is required")
|
|
}
|
|
if !errors.Is(err, os.ErrNotExist) {
|
|
return Result{}, err
|
|
}
|
|
if err = c.pending(); err != nil {
|
|
return Result{}, err
|
|
}
|
|
if err = prepare(ctx); err != nil {
|
|
return Result{}, err
|
|
}
|
|
if err = ctx.Err(); err != nil {
|
|
return Result{}, err
|
|
}
|
|
items, err := inventory(ctx, c.control)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
j := journal{Version: 1, Source: c.source, Job: job, Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
|
|
for _, s := range items {
|
|
if s.Running && !s.Healthy {
|
|
return Result{}, errors.New("test writer is not ready")
|
|
}
|
|
j.Services = append(j.Services, serviceRecord{Service: s})
|
|
}
|
|
if err = c.save(j); err != nil {
|
|
return Result{}, err
|
|
}
|
|
var workErr error
|
|
for i := range j.Services {
|
|
if !j.Services[i].Running {
|
|
continue
|
|
}
|
|
j.Services[i].StopRequested = true
|
|
if workErr = c.save(j); workErr != nil {
|
|
break
|
|
}
|
|
if workErr = c.control.Stop(ctx, j.Services[i].ID); workErr != nil {
|
|
break
|
|
}
|
|
}
|
|
if workErr == nil {
|
|
workErr = c.stopped(ctx, j)
|
|
}
|
|
if workErr == nil {
|
|
for i := range j.Services {
|
|
if j.Services[i].StopRequested {
|
|
j.Services[i].StopVerified = true
|
|
}
|
|
}
|
|
j.State = "capturing"
|
|
workErr = c.save(j)
|
|
}
|
|
if workErr == nil {
|
|
j.SnapshotID, workErr = invokeCapture(ctx, capture)
|
|
}
|
|
if workErr == nil {
|
|
workErr = ctx.Err()
|
|
}
|
|
if workErr == nil {
|
|
decoded, decodeErr := hex.DecodeString(j.SnapshotID)
|
|
if decodeErr != nil || len(decoded) != 32 || strings.ToLower(j.SnapshotID) != j.SnapshotID {
|
|
workErr = errors.New("invalid candidate snapshot digest")
|
|
}
|
|
}
|
|
if workErr == nil {
|
|
workErr = c.stopped(ctx, j)
|
|
}
|
|
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
|
|
defer cancel()
|
|
recoveryErr := c.restore(recoveryCtx, &j)
|
|
if recoveryErr != nil {
|
|
return j.Result, errors.Join(workErr, recoveryErr)
|
|
}
|
|
if workErr != nil {
|
|
j.State = "failed"
|
|
j.Code = "capture_or_isolation_failed"
|
|
} else {
|
|
j.State = "succeeded"
|
|
}
|
|
ended := time.Now().UTC()
|
|
j.FinishedAt = &ended
|
|
if err = c.save(j); err != nil {
|
|
return j.Result, err
|
|
}
|
|
return j.Result, workErr
|
|
}
|