Files
knowledge-governance/internal/librarymaintenance/gate.go
T
姚章浩 3643e9bf82
build / build (push) Successful in 1m20s
fix(library): 维护收尾必须按预备份恢复并校验
删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。
新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下
执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。
预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者;
写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
2026-10-29 00:50:32 +08:00

190 lines
6.5 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package librarymaintenance
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"time"
"knowledge-governance/internal/libraryfreeze"
)
// dockerGate 是目标入口闸门的真实实现:它只认"容器已经优雅退出"+
// 闸门自己写下的 drain.json(停止接客时还有多少在途连接、最后是否归零)。
// 不看 HTTP 探测就宣布"入口关了"——那是猜。
type dockerGate struct {
api *dockerAPI
project string
service string
drainFile string
timeout time.Duration
}
func newDockerGate(socket, project, service, drainFile string, timeout time.Duration) (*dockerGate, error) {
if project == "" || service == "" || drainFile == "" || drainFile[0] != '/' {
return nil, errors.New("入口闸门配置不完整")
}
api, err := newDockerAPI(socket, timeout)
if err != nil {
return nil, err
}
return &dockerGate{api: api, project: project, service: service, drainFile: drainFile, timeout: timeout}, nil
}
func (g *dockerGate) container(ctx context.Context) (containerState, error) {
items, err := g.api.listProject(ctx, g.project)
if err != nil {
return containerState{}, err
}
id := ""
for _, item := range items {
if item.Labels["com.docker.compose.service"] != g.service {
continue
}
if id != "" {
return containerState{}, fmt.Errorf("入口闸门实例数异常:多于一个 %s", g.service)
}
id = item.ID
}
if id == "" {
return containerState{}, fmt.Errorf("找不到入口闸门服务 %s", g.service)
}
data, err := g.api.inspect(ctx, id)
if err != nil {
return containerState{}, err
}
if data.Config.Labels["com.docker.compose.project"] != g.project ||
data.Config.Labels["com.docker.compose.service"] != g.service {
return containerState{}, errors.New("入口闸门身份不匹配")
}
return data, nil
}
// drainRecord 是闸门自己写的现场:关闭瞬间还有多少在途连接(active_at_close)、
// 等了多久(drain_ms)、一共服务过多少连接(served)。
type drainRecord struct {
Phase string `json:"phase"`
Active int `json:"active"`
ActiveAtClose int `json:"active_at_close"`
Served int `json:"served"`
DrainMS int64 `json:"drain_ms"`
UptimeMS int64 `json:"uptime_ms"`
}
func (g *dockerGate) readDrain() (drainRecord, error) {
file, err := os.Open(g.drainFile)
if err != nil {
return drainRecord{}, err
}
defer file.Close()
var record drainRecord
decoder := json.NewDecoder(io.LimitReader(file, 64<<10))
if err := decoder.Decode(&record); err != nil {
return drainRecord{}, err
}
return record, nil
}
// CloseIngress:先删掉上一次的 drain 记录(不能用旧证据冒充这次),再发 SIGTERM,
// 等到容器自己退出,最后只认 exit 0 + phase=drained。
func (g *dockerGate) CloseIngress(ctx context.Context) (libraryfreeze.GateEvidence, error) {
bounded, cancel := context.WithTimeout(ctx, g.timeout)
defer cancel()
data, err := g.container(bounded)
if err != nil {
return libraryfreeze.GateEvidence{}, err
}
closeStarted := time.Now()
if data.State.Running {
if err := os.Remove(g.drainFile); err != nil && !errors.Is(err, os.ErrNotExist) {
return libraryfreeze.GateEvidence{}, fmt.Errorf("无法清理上次的排空记录:%w", err)
}
// 删掉之后必须真的不存在:否则接下来读到的任何 drain 记录都可能是上一次的。
if _, err := os.Stat(g.drainFile); err == nil {
return libraryfreeze.GateEvidence{}, errors.New("上一次的排空记录仍然存在,不能用它冒充这次")
} else if !errors.Is(err, os.ErrNotExist) {
return libraryfreeze.GateEvidence{}, fmt.Errorf("无法确认排空记录已清理:%w", err)
}
if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil {
return libraryfreeze.GateEvidence{}, err
}
deadline := time.Now().Add(g.timeout)
for {
state, err := g.container(bounded)
if err != nil {
return libraryfreeze.GateEvidence{}, err
}
if !state.State.Running {
data = state
break
}
if time.Now().After(deadline) {
return libraryfreeze.GateEvidence{}, errors.New("入口闸门没有在期限内退出(在途请求可能还没结束)")
}
select {
case <-bounded.Done():
return libraryfreeze.GateEvidence{}, errors.New("入口闸门退出等待被中断")
case <-time.After(100 * time.Millisecond):
}
}
}
record, err := g.readDrain()
if err != nil {
return libraryfreeze.GateEvidence{}, fmt.Errorf("读不到入口排空记录:%w", err)
}
// 记录必须是这次关闭之后写的:早于 closeStarted 的文件说明读到的是历史证据。
if info, statErr := os.Stat(g.drainFile); statErr != nil {
return libraryfreeze.GateEvidence{}, fmt.Errorf("读不到排空记录文件:%w", statErr)
} else if info.ModTime().Before(closeStarted.Add(-time.Second)) {
return libraryfreeze.GateEvidence{}, errors.New("排空记录早于本次关闭,不能当作这次的证据")
}
if record.Phase != "drained" || data.State.ExitCode != 0 {
return libraryfreeze.GateEvidence{}, fmt.Errorf("入口排空未确认(phase=%s exit=%d active=%d)",
record.Phase, data.State.ExitCode, record.Active)
}
return libraryfreeze.GateEvidence{Closed: true, Drained: true, ActiveAtClose: record.ActiveAtClose,
Served: record.Served, Detail: fmt.Sprintf("gate exit=0 phase=%s drain_ms=%d active_at_close=%d",
record.Phase, record.DrainMS, record.ActiveAtClose)}, nil
}
// OpenIngress:把闸门拉起来并等健康检查通过;能不能真的服务由入口探针再确认一次。
func (g *dockerGate) OpenIngress(ctx context.Context) error {
bounded, cancel := context.WithTimeout(ctx, g.timeout)
defer cancel()
data, err := g.container(bounded)
if err != nil {
return err
}
if !data.State.Running {
if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil {
return err
}
}
deadline := time.Now().Add(g.timeout)
for {
state, err := g.container(bounded)
if err != nil {
return err
}
healthy := state.State.Running && state.State.Health != nil && state.State.Health.Status == "healthy"
if healthy {
// 记下新的 serving 记录:下一次关闭要用它自己的证据。
if _, err := g.readDrain(); err == nil {
return nil
}
return nil
}
if time.Now().After(deadline) {
return errors.New("入口闸门没有在期限内恢复健康")
}
select {
case <-bounded.Done():
return errors.New("入口闸门恢复等待被中断")
case <-time.After(200 * time.Millisecond):
}
}
}