Files
knowledge-governance/internal/libraryfreeze/freeze.go
T
姚章浩 c5e033dd0e
build / build (push) Failing after 15s
fix(library): 幂等重放、跨版本去重用量与契约复核
- 同作业号重跑读持久日志+已封存清单按首次请求重放(source_read=false,
  writers_stopped=false),换范围/绑定库或清单校验失败即拒绝
- 归档用量按跨版本去重统计内容字节,清单/未引用 blob/实际分配单列,
  不再用各版本 unique 之和冒充归档占用
- 冻结窗口内复核来源契约,不一致即中止;拒绝组外同镜像运行容器
- 补齐整库只读接口的 OpenAPI 契约
- 按用户确认取消 A06:文档/页面/接口不再把效果测试列为封版门槛
2026-10-28 12:01:09 +08:00

492 lines
13 KiB
Go

// Package libraryfreeze coordinates a test-only maintenance window. It never
// grants production authorization or starts services which were already stopped.
package libraryfreeze
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"sort"
"strings"
"syscall"
"time"
)
type Service struct {
ID string `json:"id"`
Running bool `json:"running"`
Healthy bool `json:"healthy"`
}
// Inventory must enumerate the complete, independently scoped test writer set,
// using immutable instance IDs. Never implement it as only the currently running
// subset. Start must wait for readiness; callers still read back the result.
type Controller interface {
Inventory(context.Context) ([]Service, error)
Stop(context.Context, string) error
Start(context.Context, string) error
}
type Result struct {
State string `json:"state"`
SnapshotID string `json:"snapshot_id,omitempty"`
}
type serviceRecord struct {
Service
StopRequested bool `json:"stop_requested"`
StopVerified bool `json:"stop_verified"`
Restored bool `json:"restored"`
}
type journal struct {
Version int `json:"version"`
Source string `json:"source"`
Job string `json:"job"`
Result
Services []serviceRecord `json:"services"`
StartedAt time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
Code string `json:"code,omitempty"`
}
type Coordinator struct {
root *os.Root
source string
control Controller
}
func Open(root, source string, c Controller) (*Coordinator, error) {
if !filepath.IsAbs(root) || !validID(source) || c == nil {
return nil, errors.New("invalid freeze configuration")
}
info, err := os.Lstat(root)
if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 {
return nil, errors.New("freeze directory must be private and real")
}
resolved, err := filepath.EvalSymlinks(root)
if err != nil || resolved != filepath.Clean(root) {
return nil, errors.New("freeze directory contains symlink")
}
r, err := os.OpenRoot(root)
if err != nil {
return nil, err
}
return &Coordinator{root: r, source: source, control: c}, nil
}
func (c *Coordinator) Close() error { return c.root.Close() }
func (c *Coordinator) Recover(ctx context.Context) error {
lock, err := c.lock()
if err != nil {
return err
}
defer unlock(lock)
dir, err := c.root.Open(".")
if err != nil {
return err
}
defer dir.Close()
entries, err := dir.ReadDir(-1)
if err != nil {
return err
}
// Validate every journal before issuing recovery writes. Corrupt records or
// another source's directory must not turn into broad service startup.
var pending []journal
for _, entry := range entries {
if !strings.HasSuffix(entry.Name(), ".json") {
continue
}
job := strings.TrimSuffix(entry.Name(), ".json")
if !validID(job) {
return errors.New("invalid freeze journal name")
}
j, err := c.load(job)
if err != nil {
return err
}
if !terminal(j.State) {
pending = append(pending, j)
}
}
if len(pending) > 1 {
return errors.New("multiple interrupted freezes require operator reconciliation")
}
for _, j := range pending {
if err = c.restore(ctx, &j); err != nil {
return err
}
j.State = "failed"
j.Code = "interrupted"
ended := time.Now().UTC()
j.FinishedAt = &ended
if err = c.save(j); err != nil {
return err
}
}
return nil
}
// Inspect reports the persisted result of a job, if this coordinator ever ran it.
// A caller which re-uses a job ID needs the original request identity (time,
// scope, knowledge base, contract) to replay idempotently instead of guessing a
// fresh one, which would be rejected as a conflicting request.
func (c *Coordinator) Inspect(job string) (Result, bool, error) {
if !validID(job) {
return Result{}, false, errors.New("invalid freeze job id")
}
j, err := c.load(job)
if errors.Is(err, os.ErrNotExist) {
return Result{}, false, nil
}
if err != nil {
return Result{}, false, err
}
return j.Result, true, nil
}
func invokeCapture(ctx context.Context, capture func(context.Context) (string, error)) (id string, err error) {
defer func() {
if recover() != nil {
id = ""
err = errors.New("candidate capture panicked")
}
}()
return capture(ctx)
}
func validID(s string) bool {
if len(s) == 0 || len(s) > 128 {
return false
}
for _, r := range s {
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_') {
return false
}
}
return true
}
func terminal(s string) bool { return s == "succeeded" || s == "failed" }
// A kernel lock serializes the whole test writer group, not individual KBs.
// A crash releases the lock, but leaves the write-ahead journal for recovery.
func (c *Coordinator) lock() (*os.File, error) {
f, err := c.root.OpenFile("operation.lock", os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0600)
if err != nil {
return nil, err
}
if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
f.Close()
return nil, errors.New("test source freeze is busy")
}
return f, nil
}
func unlock(f *os.File) { _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN); _ = f.Close() }
func (c *Coordinator) save(j journal) error {
data, err := json.Marshal(j)
if err != nil {
return err
}
var token [16]byte
if _, err = rand.Read(token[:]); err != nil {
return err
}
name := "write-" + hex.EncodeToString(token[:]) + ".tmp"
f, err := c.root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return err
}
defer c.root.Remove(name)
defer f.Close()
if _, err = f.Write(data); err != nil {
return err
}
if err = f.Sync(); err != nil {
return err
}
if err = f.Close(); err != nil {
return err
}
if err = c.root.Rename(name, j.Job+".json"); err != nil {
return err
}
dir, err := c.root.Open(".")
if err != nil {
return err
}
defer dir.Close()
return dir.Sync()
}
func (c *Coordinator) load(job string) (journal, error) {
f, err := c.root.OpenFile(job+".json", os.O_RDONLY|syscall.O_NOFOLLOW, 0)
if err != nil {
return journal{}, err
}
defer f.Close()
info, err := f.Stat()
if err != nil || !info.Mode().IsRegular() || info.Size() > 1024*1024 {
return journal{}, errors.New("invalid freeze journal file")
}
data, err := io.ReadAll(io.LimitReader(f, 1024*1024+1))
if err != nil {
return journal{}, err
}
var j journal
dec := json.NewDecoder(bytes.NewReader(data))
dec.DisallowUnknownFields()
if err = dec.Decode(&j); err != nil {
return journal{}, errors.New("invalid freeze journal")
}
var extra any
if dec.Decode(&extra) != io.EOF || j.Version != 1 || j.Source != c.source || j.Job != job || !validID(j.Job) {
return journal{}, errors.New("freeze journal identity mismatch")
}
switch j.State {
case "stopping", "capturing", "restoring", "succeeded", "failed":
default:
return journal{}, errors.New("invalid freeze journal state")
}
if len(j.Services) == 0 || len(j.Services) > 256 {
return journal{}, errors.New("invalid freeze journal services")
}
for i, s := range j.Services {
if !validID(s.ID) || (i > 0 && j.Services[i-1].ID >= s.ID) || s.StopRequested && !s.Running || s.Restored && (!s.StopRequested || !s.Running) {
return journal{}, errors.New("invalid freeze service ownership")
}
}
return j, nil
}
func (c *Coordinator) pending() error {
dir, err := c.root.Open(".")
if err != nil {
return err
}
defer dir.Close()
entries, err := dir.ReadDir(-1)
if err != nil {
return err
}
for _, entry := range entries {
if strings.HasSuffix(entry.Name(), ".json") {
job := strings.TrimSuffix(entry.Name(), ".json")
if !validID(job) {
return errors.New("invalid freeze journal name")
}
j, err := c.load(job)
if err != nil {
return err
}
if !terminal(j.State) {
return errors.New("interrupted test freeze requires recovery")
}
}
}
return nil
}
func inventory(ctx context.Context, control Controller) ([]Service, error) {
items, err := control.Inventory(ctx)
if err != nil {
return nil, errors.New("test writer inventory unavailable")
}
if len(items) == 0 || len(items) > 256 {
return nil, errors.New("invalid test writer inventory")
}
seen := map[string]bool{}
for _, s := range items {
if !validID(s.ID) || seen[s.ID] {
return nil, errors.New("invalid test writer identity")
}
seen[s.ID] = true
}
sort.Slice(items, func(i, j int) bool { return items[i].ID < items[j].ID })
return items, nil
}
func (c *Coordinator) stopped(ctx context.Context, j journal) error {
items, err := inventory(ctx, c.control)
if err != nil {
return err
}
if len(items) != len(j.Services) {
return errors.New("test writer topology changed")
}
for i, s := range items {
if s.ID != j.Services[i].ID || s.Running {
return errors.New("test writers not isolated")
}
}
return nil
}
func (c *Coordinator) restore(ctx context.Context, j *journal) error {
j.State = "restoring"
// Restoring services is still necessary if the journal disk fails after a
// persisted stop intent. Keep the error, but do not let it suppress recovery.
var failures error = c.save(*j)
for i := range j.Services {
s := &j.Services[i]
if !s.Running || !s.StopRequested {
continue
}
items, err := inventory(ctx, c.control)
if err != nil {
failures = errors.Join(failures, err)
continue
}
found := false
for _, actual := range items {
if actual.ID != s.ID {
continue
}
found = true
if actual.Running && !s.StopVerified {
failures = errors.Join(failures, errors.New("stop outcome remains uncertain; service may still stop later"))
break
}
if !actual.Running {
s.StopVerified = true
if saveErr := c.save(*j); saveErr != nil {
failures = errors.Join(failures, saveErr)
}
if err = c.control.Start(ctx, s.ID); err != nil {
failures = errors.Join(failures, errors.New("test service restart failed"))
break
}
}
after, readErr := inventory(ctx, c.control)
healthy := false
if readErr == nil {
for _, a := range after {
if a.ID == s.ID && a.Running && a.Healthy {
healthy = true
}
}
}
if !healthy {
failures = errors.Join(failures, errors.New("test service health unverified"))
break
}
s.Restored = true
if err = c.save(*j); err != nil {
failures = errors.Join(failures, err)
}
break
}
if !found {
failures = errors.Join(failures, errors.New("original test service missing; replacement not started"))
}
}
return failures
}
func (c *Coordinator) Run(ctx context.Context, job string, capture func(context.Context) (string, error)) (Result, error) {
return c.RunPrepared(ctx, job, func(context.Context) error { return nil }, capture)
}
// RunPrepared runs readiness checks under the source lock, before any stop
// intent. It does not repeat checks or capture for an already completed job.
func (c *Coordinator) RunPrepared(ctx context.Context, job string, prepare func(context.Context) error, capture func(context.Context) (string, error)) (Result, error) {
if !validID(job) || capture == nil || prepare == nil {
return Result{}, errors.New("invalid freeze request")
}
if err := ctx.Err(); err != nil {
return Result{}, err
}
lock, err := c.lock()
if err != nil {
return Result{}, err
}
defer unlock(lock)
previous, err := c.load(job)
if err == nil {
if previous.State == "succeeded" {
return previous.Result, nil
}
return previous.Result, errors.New("freeze request already exists; recovery or a new request is required")
}
if !errors.Is(err, os.ErrNotExist) {
return Result{}, err
}
if err = c.pending(); err != nil {
return Result{}, err
}
if err = prepare(ctx); err != nil {
return Result{}, err
}
if err = ctx.Err(); err != nil {
return Result{}, err
}
items, err := inventory(ctx, c.control)
if err != nil {
return Result{}, err
}
j := journal{Version: 1, Source: c.source, Job: job, Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
for _, s := range items {
if s.Running && !s.Healthy {
return Result{}, errors.New("test writer is not ready")
}
j.Services = append(j.Services, serviceRecord{Service: s})
}
if err = c.save(j); err != nil {
return Result{}, err
}
var workErr error
for i := range j.Services {
if !j.Services[i].Running {
continue
}
j.Services[i].StopRequested = true
if workErr = c.save(j); workErr != nil {
break
}
if workErr = c.control.Stop(ctx, j.Services[i].ID); workErr != nil {
break
}
}
if workErr == nil {
workErr = c.stopped(ctx, j)
}
if workErr == nil {
for i := range j.Services {
if j.Services[i].StopRequested {
j.Services[i].StopVerified = true
}
}
j.State = "capturing"
workErr = c.save(j)
}
if workErr == nil {
j.SnapshotID, workErr = invokeCapture(ctx, capture)
}
if workErr == nil {
workErr = ctx.Err()
}
if workErr == nil {
decoded, decodeErr := hex.DecodeString(j.SnapshotID)
if decodeErr != nil || len(decoded) != 32 || strings.ToLower(j.SnapshotID) != j.SnapshotID {
workErr = errors.New("invalid candidate snapshot digest")
}
}
if workErr == nil {
workErr = c.stopped(ctx, j)
}
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
defer cancel()
recoveryErr := c.restore(recoveryCtx, &j)
if recoveryErr != nil {
return j.Result, errors.Join(workErr, recoveryErr)
}
if workErr != nil {
j.State = "failed"
j.Code = "capture_or_isolation_failed"
} else {
j.State = "succeeded"
}
ended := time.Now().UTC()
j.FinishedAt = &ended
if err = c.save(j); err != nil {
return j.Result, err
}
return j.Result, workErr
}