3643e9bf82
build / build (push) Successful in 1m20s
删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。 新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下 执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。 预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者; 写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
190 lines
6.5 KiB
Go
190 lines
6.5 KiB
Go
package librarymaintenance
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/libraryfreeze"
|
||
)
|
||
|
||
// dockerGate 是目标入口闸门的真实实现:它只认"容器已经优雅退出"+
|
||
// 闸门自己写下的 drain.json(停止接客时还有多少在途连接、最后是否归零)。
|
||
// 不看 HTTP 探测就宣布"入口关了"——那是猜。
|
||
type dockerGate struct {
|
||
api *dockerAPI
|
||
project string
|
||
service string
|
||
drainFile string
|
||
timeout time.Duration
|
||
}
|
||
|
||
func newDockerGate(socket, project, service, drainFile string, timeout time.Duration) (*dockerGate, error) {
|
||
if project == "" || service == "" || drainFile == "" || drainFile[0] != '/' {
|
||
return nil, errors.New("入口闸门配置不完整")
|
||
}
|
||
api, err := newDockerAPI(socket, timeout)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return &dockerGate{api: api, project: project, service: service, drainFile: drainFile, timeout: timeout}, nil
|
||
}
|
||
|
||
func (g *dockerGate) container(ctx context.Context) (containerState, error) {
|
||
items, err := g.api.listProject(ctx, g.project)
|
||
if err != nil {
|
||
return containerState{}, err
|
||
}
|
||
id := ""
|
||
for _, item := range items {
|
||
if item.Labels["com.docker.compose.service"] != g.service {
|
||
continue
|
||
}
|
||
if id != "" {
|
||
return containerState{}, fmt.Errorf("入口闸门实例数异常:多于一个 %s", g.service)
|
||
}
|
||
id = item.ID
|
||
}
|
||
if id == "" {
|
||
return containerState{}, fmt.Errorf("找不到入口闸门服务 %s", g.service)
|
||
}
|
||
data, err := g.api.inspect(ctx, id)
|
||
if err != nil {
|
||
return containerState{}, err
|
||
}
|
||
if data.Config.Labels["com.docker.compose.project"] != g.project ||
|
||
data.Config.Labels["com.docker.compose.service"] != g.service {
|
||
return containerState{}, errors.New("入口闸门身份不匹配")
|
||
}
|
||
return data, nil
|
||
}
|
||
|
||
// drainRecord 是闸门自己写的现场:关闭瞬间还有多少在途连接(active_at_close)、
|
||
// 等了多久(drain_ms)、一共服务过多少连接(served)。
|
||
type drainRecord struct {
|
||
Phase string `json:"phase"`
|
||
Active int `json:"active"`
|
||
ActiveAtClose int `json:"active_at_close"`
|
||
Served int `json:"served"`
|
||
DrainMS int64 `json:"drain_ms"`
|
||
UptimeMS int64 `json:"uptime_ms"`
|
||
}
|
||
|
||
func (g *dockerGate) readDrain() (drainRecord, error) {
|
||
file, err := os.Open(g.drainFile)
|
||
if err != nil {
|
||
return drainRecord{}, err
|
||
}
|
||
defer file.Close()
|
||
var record drainRecord
|
||
decoder := json.NewDecoder(io.LimitReader(file, 64<<10))
|
||
if err := decoder.Decode(&record); err != nil {
|
||
return drainRecord{}, err
|
||
}
|
||
return record, nil
|
||
}
|
||
|
||
// CloseIngress:先删掉上一次的 drain 记录(不能用旧证据冒充这次),再发 SIGTERM,
|
||
// 等到容器自己退出,最后只认 exit 0 + phase=drained。
|
||
func (g *dockerGate) CloseIngress(ctx context.Context) (libraryfreeze.GateEvidence, error) {
|
||
bounded, cancel := context.WithTimeout(ctx, g.timeout)
|
||
defer cancel()
|
||
data, err := g.container(bounded)
|
||
if err != nil {
|
||
return libraryfreeze.GateEvidence{}, err
|
||
}
|
||
closeStarted := time.Now()
|
||
if data.State.Running {
|
||
if err := os.Remove(g.drainFile); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||
return libraryfreeze.GateEvidence{}, fmt.Errorf("无法清理上次的排空记录:%w", err)
|
||
}
|
||
// 删掉之后必须真的不存在:否则接下来读到的任何 drain 记录都可能是上一次的。
|
||
if _, err := os.Stat(g.drainFile); err == nil {
|
||
return libraryfreeze.GateEvidence{}, errors.New("上一次的排空记录仍然存在,不能用它冒充这次")
|
||
} else if !errors.Is(err, os.ErrNotExist) {
|
||
return libraryfreeze.GateEvidence{}, fmt.Errorf("无法确认排空记录已清理:%w", err)
|
||
}
|
||
if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil {
|
||
return libraryfreeze.GateEvidence{}, err
|
||
}
|
||
deadline := time.Now().Add(g.timeout)
|
||
for {
|
||
state, err := g.container(bounded)
|
||
if err != nil {
|
||
return libraryfreeze.GateEvidence{}, err
|
||
}
|
||
if !state.State.Running {
|
||
data = state
|
||
break
|
||
}
|
||
if time.Now().After(deadline) {
|
||
return libraryfreeze.GateEvidence{}, errors.New("入口闸门没有在期限内退出(在途请求可能还没结束)")
|
||
}
|
||
select {
|
||
case <-bounded.Done():
|
||
return libraryfreeze.GateEvidence{}, errors.New("入口闸门退出等待被中断")
|
||
case <-time.After(100 * time.Millisecond):
|
||
}
|
||
}
|
||
}
|
||
record, err := g.readDrain()
|
||
if err != nil {
|
||
return libraryfreeze.GateEvidence{}, fmt.Errorf("读不到入口排空记录:%w", err)
|
||
}
|
||
// 记录必须是这次关闭之后写的:早于 closeStarted 的文件说明读到的是历史证据。
|
||
if info, statErr := os.Stat(g.drainFile); statErr != nil {
|
||
return libraryfreeze.GateEvidence{}, fmt.Errorf("读不到排空记录文件:%w", statErr)
|
||
} else if info.ModTime().Before(closeStarted.Add(-time.Second)) {
|
||
return libraryfreeze.GateEvidence{}, errors.New("排空记录早于本次关闭,不能当作这次的证据")
|
||
}
|
||
if record.Phase != "drained" || data.State.ExitCode != 0 {
|
||
return libraryfreeze.GateEvidence{}, fmt.Errorf("入口排空未确认(phase=%s exit=%d active=%d)",
|
||
record.Phase, data.State.ExitCode, record.Active)
|
||
}
|
||
return libraryfreeze.GateEvidence{Closed: true, Drained: true, ActiveAtClose: record.ActiveAtClose,
|
||
Served: record.Served, Detail: fmt.Sprintf("gate exit=0 phase=%s drain_ms=%d active_at_close=%d",
|
||
record.Phase, record.DrainMS, record.ActiveAtClose)}, nil
|
||
}
|
||
|
||
// OpenIngress:把闸门拉起来并等健康检查通过;能不能真的服务由入口探针再确认一次。
|
||
func (g *dockerGate) OpenIngress(ctx context.Context) error {
|
||
bounded, cancel := context.WithTimeout(ctx, g.timeout)
|
||
defer cancel()
|
||
data, err := g.container(bounded)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !data.State.Running {
|
||
if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
deadline := time.Now().Add(g.timeout)
|
||
for {
|
||
state, err := g.container(bounded)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
healthy := state.State.Running && state.State.Health != nil && state.State.Health.Status == "healthy"
|
||
if healthy {
|
||
// 记下新的 serving 记录:下一次关闭要用它自己的证据。
|
||
if _, err := g.readDrain(); err == nil {
|
||
return nil
|
||
}
|
||
return nil
|
||
}
|
||
if time.Now().After(deadline) {
|
||
return errors.New("入口闸门没有在期限内恢复健康")
|
||
}
|
||
select {
|
||
case <-bounded.Done():
|
||
return errors.New("入口闸门恢复等待被中断")
|
||
case <-time.After(200 * time.Millisecond):
|
||
}
|
||
}
|
||
}
|