403 lines
13 KiB
Go
403 lines
13 KiB
Go
// Package librarymaintenance 把"目标维护窗口"落成一个可验证的实例级动作:
|
||
// 关闭目标入口 → 停下全部声明的写入者 → 验证在途排空 → 交给发布链路写目标
|
||
// → 恢复原本在跑的服务与入口。
|
||
//
|
||
// 它不产生授权、不读生产凭据,也不把"只锁了治理库 scope"当成系统维护。
|
||
package librarymaintenance
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/libraryfreeze"
|
||
)
|
||
|
||
// LevelInstance 表示写入者被共享给整个实例:停它影响所有知识库,
|
||
// 因此窗口必须按"实例级维护"对外标注,而不是"某个库在维护"。
|
||
const LevelInstance = "instance"
|
||
|
||
// Target 是目标的精确身份:实例、入口、写入者、要写的库/索引/桶。
|
||
// Fingerprint 覆盖全部字段,用来发现"配置换了但作业还按旧目标跑"。
|
||
type Target struct {
|
||
Instance string `json:"instance"`
|
||
Level string `json:"level"`
|
||
Project string `json:"project,omitempty"`
|
||
Writers []string `json:"writers,omitempty"`
|
||
Ingress string `json:"ingress,omitempty"`
|
||
Database string `json:"database"`
|
||
Index string `json:"index"`
|
||
Bucket string `json:"bucket"`
|
||
Fingerprint string `json:"fingerprint,omitempty"`
|
||
}
|
||
|
||
// Config 来自部署注入的配置。任何一项缺失都不许"降级成库级维护"蒙过去。
|
||
type Config struct {
|
||
Instance string
|
||
Project string
|
||
Writers []string
|
||
NonWriters map[string]string
|
||
DockerSocket string
|
||
IngressURL string
|
||
// IngressService 是入口闸门在 compose 项目里的服务名;DrainFile 是它写
|
||
// 排空证据的宿主机文件。两者缺一,就只能说"停了写入者",不能说"排空了"。
|
||
IngressService string
|
||
IngressDrainFile string
|
||
Database string
|
||
Index string
|
||
Bucket string
|
||
// 后台静默检查需要的运行时依赖(不写进维护记录)。
|
||
DatabaseURL string
|
||
RedisAddr string
|
||
RedisPassword string
|
||
RedisDB int
|
||
Timeout time.Duration
|
||
}
|
||
|
||
type Manager struct {
|
||
target Target
|
||
source string
|
||
docker string
|
||
writers []string
|
||
nonWriters map[string]string
|
||
timeout time.Duration
|
||
client *http.Client
|
||
gate *dockerGate
|
||
api *dockerAPI
|
||
quiescer *targetQuiescer
|
||
}
|
||
|
||
// Close 释放拓扑/静默检查持有的连接。
|
||
func (m *Manager) Close() {
|
||
if m.api != nil {
|
||
m.api.Close()
|
||
}
|
||
if m.quiescer != nil {
|
||
m.quiescer.Close()
|
||
}
|
||
}
|
||
|
||
func New(cfg Config) (*Manager, error) {
|
||
instance := strings.TrimSpace(cfg.Instance)
|
||
project := strings.TrimSpace(cfg.Project)
|
||
ingress := strings.TrimSpace(cfg.IngressURL)
|
||
if instance == "" || project == "" {
|
||
return nil, errors.New("实例级维护配置缺少 LIBRARY_TARGET_INSTANCE/PROJECT")
|
||
}
|
||
if len(cfg.Writers) == 0 {
|
||
return nil, errors.New("实例级维护配置缺少写入者声明 LIBRARY_TARGET_WRITERS")
|
||
}
|
||
if !filepath.IsAbs(cfg.DockerSocket) {
|
||
return nil, errors.New("Docker socket 必须是绝对路径")
|
||
}
|
||
parsed, err := url.Parse(ingress)
|
||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.Path == "" || parsed.User != nil {
|
||
return nil, errors.New("目标入口地址无效(LIBRARY_TARGET_INGRESS_URL)")
|
||
}
|
||
if cfg.Database == "" || cfg.Index == "" || cfg.Bucket == "" {
|
||
return nil, errors.New("目标身份不完整:需要数据库/索引/存储桶")
|
||
}
|
||
if cfg.Timeout <= 0 {
|
||
cfg.Timeout = 2 * time.Minute
|
||
}
|
||
writers := append([]string(nil), cfg.Writers...)
|
||
sort.Strings(writers)
|
||
target := Target{Instance: instance, Level: LevelInstance, Project: project, Writers: writers,
|
||
Ingress: ingress, Database: cfg.Database, Index: cfg.Index, Bucket: cfg.Bucket}
|
||
fingerprint, err := fingerprintOf(target)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
target.Fingerprint = fingerprint
|
||
api, err := newDockerAPI(cfg.DockerSocket, cfg.Timeout)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
manager := &Manager{target: target, source: project, docker: cfg.DockerSocket,
|
||
writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout,
|
||
client: &http.Client{Timeout: 5 * time.Second}, api: api}
|
||
if cfg.IngressService != "" || cfg.IngressDrainFile != "" {
|
||
gate, err := newDockerGate(cfg.DockerSocket, project, cfg.IngressService, cfg.IngressDrainFile, cfg.Timeout)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
manager.gate = gate
|
||
}
|
||
if cfg.DatabaseURL != "" && cfg.RedisAddr != "" {
|
||
quiescer, err := newTargetQuiescer(context.Background(), cfg.DatabaseURL, cfg.RedisAddr,
|
||
cfg.RedisPassword, cfg.RedisDB, cfg.Timeout)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
manager.quiescer = quiescer
|
||
}
|
||
return manager, nil
|
||
}
|
||
|
||
func fingerprintOf(t Target) (string, error) {
|
||
canonical := struct {
|
||
Instance string `json:"instance"`
|
||
Level string `json:"level"`
|
||
Project string `json:"project"`
|
||
Writers []string `json:"writers"`
|
||
Ingress string `json:"ingress"`
|
||
Database string `json:"database"`
|
||
Index string `json:"index"`
|
||
Bucket string `json:"bucket"`
|
||
}{t.Instance, t.Level, t.Project, t.Writers, t.Ingress, t.Database, t.Index, t.Bucket}
|
||
raw, err := json.Marshal(canonical)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(raw)
|
||
return hex.EncodeToString(sum[:]), nil
|
||
}
|
||
|
||
func (m *Manager) Target() Target { return m.target }
|
||
|
||
// Description 是写进维护记录的一句话身份,含指纹,便于事后核对"当时动的是谁"。
|
||
func (m *Manager) Description() string {
|
||
t := m.target
|
||
return fmt.Sprintf("level=%s instance=%s project=%s writers=%s ingress=%s db=%s index=%s bucket=%s fingerprint=%s",
|
||
t.Level, t.Instance, t.Project, strings.Join(t.Writers, "+"), t.Ingress, t.Database, t.Index, t.Bucket, t.Fingerprint)
|
||
}
|
||
|
||
// Open 返回绑定到该目标的协调器与控制器。调用方负责 Close。
|
||
func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.ProjectController, error) {
|
||
control, err := libraryfreeze.NewProjectController(m.docker, m.source, m.writers, m.nonWriters, m.timeout)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
coordinator, err := libraryfreeze.Open(root, m.source, control)
|
||
if err != nil {
|
||
control.Close()
|
||
return nil, nil, err
|
||
}
|
||
if err := coordinator.SetTarget(LevelInstance, m.target.Ingress, m.ingressClosed, m.ingressReady); err != nil {
|
||
coordinator.Close()
|
||
control.Close()
|
||
return nil, nil, err
|
||
}
|
||
if err := coordinator.SetTargetFingerprint(m.target.Fingerprint); err != nil {
|
||
coordinator.Close()
|
||
control.Close()
|
||
return nil, nil, err
|
||
}
|
||
if m.gate != nil {
|
||
if err := coordinator.SetGate(m.gate); err != nil {
|
||
coordinator.Close()
|
||
control.Close()
|
||
return nil, nil, err
|
||
}
|
||
}
|
||
// 后台静默检查是实例级窗口的必备条件:没有它就等于只排空了 HTTP。
|
||
if m.quiescer == nil {
|
||
coordinator.Close()
|
||
control.Close()
|
||
return nil, nil, errors.New("缺少后台静默检查(目标库/队列未配置),拒绝开实例级窗口")
|
||
}
|
||
if err := coordinator.SetQuiescer(m.quiescer); err != nil {
|
||
coordinator.Close()
|
||
control.Close()
|
||
return nil, nil, err
|
||
}
|
||
return coordinator, control, nil
|
||
}
|
||
|
||
// OriginalState 是"动之前"的原服务状态,落在维护记录里:
|
||
// 只按它恢复,绝不顺手把原本没跑的服务拉起来。
|
||
func (m *Manager) OriginalState(ctx context.Context) (string, error) {
|
||
control, err := libraryfreeze.NewProjectController(m.docker, m.source, m.writers, m.nonWriters, m.timeout)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer control.Close()
|
||
items, err := control.Inventory(ctx)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
parts := make([]string, 0, len(items))
|
||
for _, item := range items {
|
||
parts = append(parts, fmt.Sprintf("%s=%s running=%t healthy=%t", m.serviceOf(item.ID), item.ID, item.Running, item.Healthy))
|
||
}
|
||
sort.Strings(parts)
|
||
return strings.Join(parts, "; "), nil
|
||
}
|
||
|
||
func (m *Manager) serviceOf(id string) string {
|
||
return "container:" + id
|
||
}
|
||
|
||
// VerifyRunning 确认目标入口真的在服务(部署与恢复后都要看,不看容器状态)。
|
||
func (m *Manager) VerifyRunning(ctx context.Context) error {
|
||
return m.ingressReady(ctx)
|
||
}
|
||
|
||
// ingressClosed:入口关掉的证据是"连不上",不是"返回错误页"——
|
||
// 502/503 说明还有一层代理在对外服务,那不是维护窗口。
|
||
func (m *Manager) ingressClosed(ctx context.Context) error {
|
||
deadline := time.Now().Add(m.timeout)
|
||
var last error
|
||
for {
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.target.Ingress, nil)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
// Maintenance probes must not leave keep-alive connections in the TCP
|
||
// ingress: the probe itself would otherwise prevent draining.
|
||
req.Close = true
|
||
resp, err := m.client.Do(req)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
||
resp.Body.Close()
|
||
last = fmt.Errorf("入口仍返回 HTTP %d", resp.StatusCode)
|
||
if time.Now().After(deadline) {
|
||
return last
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return last
|
||
case <-time.After(200 * time.Millisecond):
|
||
}
|
||
}
|
||
}
|
||
|
||
// ingressReady:入口回来 = 真的能返回 200,而不是"容器起来了"。
|
||
func (m *Manager) ingressReady(ctx context.Context) error {
|
||
deadline := time.Now().Add(m.timeout)
|
||
var last error
|
||
for {
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.target.Ingress, nil)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
// Maintenance probes must not leave keep-alive connections in the TCP
|
||
// ingress: the probe itself would otherwise prevent draining.
|
||
req.Close = true
|
||
resp, err := m.client.Do(req)
|
||
if err == nil {
|
||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||
resp.Body.Close()
|
||
if resp.StatusCode == http.StatusOK {
|
||
return nil
|
||
}
|
||
last = fmt.Errorf("入口返回 HTTP %d:%s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||
} else {
|
||
last = err
|
||
}
|
||
if time.Now().After(deadline) {
|
||
return fmt.Errorf("目标入口没有恢复:%w", last)
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return last
|
||
case <-time.After(500 * time.Millisecond):
|
||
}
|
||
}
|
||
}
|
||
|
||
// DatabaseIdentity 把连接串收敛成"host:port/db"这种可以写进维护记录的身份:
|
||
// 维护记录里不落库凭据,但必须能说清动的是哪个库。
|
||
func DatabaseIdentity(raw string) (string, error) {
|
||
parsed, err := url.Parse(strings.TrimSpace(raw))
|
||
if err != nil || parsed.Host == "" {
|
||
return "", errors.New("目标数据库连接串无效")
|
||
}
|
||
database := strings.TrimPrefix(parsed.Path, "/")
|
||
if database == "" {
|
||
return "", errors.New("目标数据库连接串缺少库名")
|
||
}
|
||
return parsed.Host + "/" + database, nil
|
||
}
|
||
|
||
const composeServiceLabel = "com.docker.compose.service"
|
||
|
||
// VerifyTopology 强制"入口唯一":
|
||
// - 写入者不许发布任何宿主机端口(否则就存在绕闸直连,维护窗口覆盖不到);
|
||
// - 入口闸门必须存在、必须是独立服务,并且发布了入口 URL 用的那个端口。
|
||
//
|
||
// 任何一条不满足都拒绝开窗口——拓扑错了,"维护"就只是关了一半的门。
|
||
func (m *Manager) VerifyTopology(ctx context.Context) error {
|
||
items, err := m.api.listProject(ctx, m.target.Project)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
byService := make(map[string]containerSummary, len(items))
|
||
for _, item := range items {
|
||
name := item.Labels[composeServiceLabel]
|
||
if name == "" {
|
||
return errors.New("项目里有容器缺少 compose service 标签,无法确认写入者边界")
|
||
}
|
||
if _, duplicated := byService[name]; duplicated {
|
||
return fmt.Errorf("项目里服务 %s 有多个实例,无法确认写入者边界", name)
|
||
}
|
||
byService[name] = item
|
||
}
|
||
for _, writer := range m.writers {
|
||
item, ok := byService[writer]
|
||
if !ok {
|
||
return fmt.Errorf("声明的写入者 %s 不存在", writer)
|
||
}
|
||
state, err := m.api.inspect(ctx, item.ID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if ports := state.publishedPorts(); len(ports) > 0 {
|
||
return fmt.Errorf("写入者 %s 直接发布了端口 %s:存在绕开入口闸门的直连通道,拒绝开维护窗口",
|
||
writer, strings.Join(ports, ","))
|
||
}
|
||
}
|
||
if m.gate == nil {
|
||
return errors.New("实例级维护必须声明入口闸门")
|
||
}
|
||
for _, writer := range m.writers {
|
||
if writer == m.gate.service {
|
||
return errors.New("入口闸门不能同时被声明为写入者")
|
||
}
|
||
}
|
||
gateItem, ok := byService[m.gate.service]
|
||
if !ok {
|
||
return fmt.Errorf("声明的入口闸门 %s 不存在", m.gate.service)
|
||
}
|
||
gateState, err := m.api.inspect(ctx, gateItem.ID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
published := gateState.publishedPorts()
|
||
if len(published) == 0 {
|
||
return fmt.Errorf("入口闸门 %s 没有发布任何端口:外部无法访问,入口定义不成立", m.gate.service)
|
||
}
|
||
// 入口 URL 的端口必须真的是这个闸门发布的端口。
|
||
parsed, err := url.Parse(m.target.Ingress)
|
||
if err != nil || parsed.Host == "" {
|
||
return errors.New("目标入口地址无效")
|
||
}
|
||
hostPort := parsed.Port()
|
||
if hostPort == "" {
|
||
if parsed.Scheme == "https" {
|
||
hostPort = "443"
|
||
} else {
|
||
hostPort = "80"
|
||
}
|
||
}
|
||
for _, entry := range published {
|
||
if strings.Contains(entry, ":"+hostPort+"->") {
|
||
return nil
|
||
}
|
||
}
|
||
return fmt.Errorf("入口 %s 的端口 %s 不是闸门发布的端口(%s):入口指向了别的服务",
|
||
m.target.Ingress, hostPort, strings.Join(published, ","))
|
||
}
|