Files
knowledge-governance/internal/librarymaintenance/target.go
T
姚章浩 286f1fb375
build / build (push) Successful in 1m23s
fix(maintenance): 避免健康探针连接阻塞入口排空
2026-10-31 03:17:45 +08:00

403 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package librarymaintenance 把"目标维护窗口"落成一个可验证的实例级动作:
// 关闭目标入口 → 停下全部声明的写入者 → 验证在途排空 → 交给发布链路写目标
// → 恢复原本在跑的服务与入口。
//
// 它不产生授权、不读生产凭据,也不把"只锁了治理库 scope"当成系统维护。
package librarymaintenance
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path/filepath"
"sort"
"strings"
"time"
"knowledge-governance/internal/libraryfreeze"
)
// LevelInstance 表示写入者被共享给整个实例:停它影响所有知识库,
// 因此窗口必须按"实例级维护"对外标注,而不是"某个库在维护"。
const LevelInstance = "instance"
// Target 是目标的精确身份:实例、入口、写入者、要写的库/索引/桶。
// Fingerprint 覆盖全部字段,用来发现"配置换了但作业还按旧目标跑"。
type Target struct {
Instance string `json:"instance"`
Level string `json:"level"`
Project string `json:"project,omitempty"`
Writers []string `json:"writers,omitempty"`
Ingress string `json:"ingress,omitempty"`
Database string `json:"database"`
Index string `json:"index"`
Bucket string `json:"bucket"`
Fingerprint string `json:"fingerprint,omitempty"`
}
// Config 来自部署注入的配置。任何一项缺失都不许"降级成库级维护"蒙过去。
type Config struct {
Instance string
Project string
Writers []string
NonWriters map[string]string
DockerSocket string
IngressURL string
// IngressService 是入口闸门在 compose 项目里的服务名;DrainFile 是它写
// 排空证据的宿主机文件。两者缺一,就只能说"停了写入者",不能说"排空了"。
IngressService string
IngressDrainFile string
Database string
Index string
Bucket string
// 后台静默检查需要的运行时依赖(不写进维护记录)。
DatabaseURL string
RedisAddr string
RedisPassword string
RedisDB int
Timeout time.Duration
}
type Manager struct {
target Target
source string
docker string
writers []string
nonWriters map[string]string
timeout time.Duration
client *http.Client
gate *dockerGate
api *dockerAPI
quiescer *targetQuiescer
}
// Close 释放拓扑/静默检查持有的连接。
func (m *Manager) Close() {
if m.api != nil {
m.api.Close()
}
if m.quiescer != nil {
m.quiescer.Close()
}
}
func New(cfg Config) (*Manager, error) {
instance := strings.TrimSpace(cfg.Instance)
project := strings.TrimSpace(cfg.Project)
ingress := strings.TrimSpace(cfg.IngressURL)
if instance == "" || project == "" {
return nil, errors.New("实例级维护配置缺少 LIBRARY_TARGET_INSTANCE/PROJECT")
}
if len(cfg.Writers) == 0 {
return nil, errors.New("实例级维护配置缺少写入者声明 LIBRARY_TARGET_WRITERS")
}
if !filepath.IsAbs(cfg.DockerSocket) {
return nil, errors.New("Docker socket 必须是绝对路径")
}
parsed, err := url.Parse(ingress)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.Path == "" || parsed.User != nil {
return nil, errors.New("目标入口地址无效(LIBRARY_TARGET_INGRESS_URL)")
}
if cfg.Database == "" || cfg.Index == "" || cfg.Bucket == "" {
return nil, errors.New("目标身份不完整:需要数据库/索引/存储桶")
}
if cfg.Timeout <= 0 {
cfg.Timeout = 2 * time.Minute
}
writers := append([]string(nil), cfg.Writers...)
sort.Strings(writers)
target := Target{Instance: instance, Level: LevelInstance, Project: project, Writers: writers,
Ingress: ingress, Database: cfg.Database, Index: cfg.Index, Bucket: cfg.Bucket}
fingerprint, err := fingerprintOf(target)
if err != nil {
return nil, err
}
target.Fingerprint = fingerprint
api, err := newDockerAPI(cfg.DockerSocket, cfg.Timeout)
if err != nil {
return nil, err
}
manager := &Manager{target: target, source: project, docker: cfg.DockerSocket,
writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout,
client: &http.Client{Timeout: 5 * time.Second}, api: api}
if cfg.IngressService != "" || cfg.IngressDrainFile != "" {
gate, err := newDockerGate(cfg.DockerSocket, project, cfg.IngressService, cfg.IngressDrainFile, cfg.Timeout)
if err != nil {
return nil, err
}
manager.gate = gate
}
if cfg.DatabaseURL != "" && cfg.RedisAddr != "" {
quiescer, err := newTargetQuiescer(context.Background(), cfg.DatabaseURL, cfg.RedisAddr,
cfg.RedisPassword, cfg.RedisDB, cfg.Timeout)
if err != nil {
return nil, err
}
manager.quiescer = quiescer
}
return manager, nil
}
func fingerprintOf(t Target) (string, error) {
canonical := struct {
Instance string `json:"instance"`
Level string `json:"level"`
Project string `json:"project"`
Writers []string `json:"writers"`
Ingress string `json:"ingress"`
Database string `json:"database"`
Index string `json:"index"`
Bucket string `json:"bucket"`
}{t.Instance, t.Level, t.Project, t.Writers, t.Ingress, t.Database, t.Index, t.Bucket}
raw, err := json.Marshal(canonical)
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
func (m *Manager) Target() Target { return m.target }
// Description 是写进维护记录的一句话身份,含指纹,便于事后核对"当时动的是谁"。
func (m *Manager) Description() string {
t := m.target
return fmt.Sprintf("level=%s instance=%s project=%s writers=%s ingress=%s db=%s index=%s bucket=%s fingerprint=%s",
t.Level, t.Instance, t.Project, strings.Join(t.Writers, "+"), t.Ingress, t.Database, t.Index, t.Bucket, t.Fingerprint)
}
// Open 返回绑定到该目标的协调器与控制器。调用方负责 Close。
func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.ProjectController, error) {
control, err := libraryfreeze.NewProjectController(m.docker, m.source, m.writers, m.nonWriters, m.timeout)
if err != nil {
return nil, nil, err
}
coordinator, err := libraryfreeze.Open(root, m.source, control)
if err != nil {
control.Close()
return nil, nil, err
}
if err := coordinator.SetTarget(LevelInstance, m.target.Ingress, m.ingressClosed, m.ingressReady); err != nil {
coordinator.Close()
control.Close()
return nil, nil, err
}
if err := coordinator.SetTargetFingerprint(m.target.Fingerprint); err != nil {
coordinator.Close()
control.Close()
return nil, nil, err
}
if m.gate != nil {
if err := coordinator.SetGate(m.gate); err != nil {
coordinator.Close()
control.Close()
return nil, nil, err
}
}
// 后台静默检查是实例级窗口的必备条件:没有它就等于只排空了 HTTP。
if m.quiescer == nil {
coordinator.Close()
control.Close()
return nil, nil, errors.New("缺少后台静默检查(目标库/队列未配置),拒绝开实例级窗口")
}
if err := coordinator.SetQuiescer(m.quiescer); err != nil {
coordinator.Close()
control.Close()
return nil, nil, err
}
return coordinator, control, nil
}
// OriginalState 是"动之前"的原服务状态,落在维护记录里:
// 只按它恢复,绝不顺手把原本没跑的服务拉起来。
func (m *Manager) OriginalState(ctx context.Context) (string, error) {
control, err := libraryfreeze.NewProjectController(m.docker, m.source, m.writers, m.nonWriters, m.timeout)
if err != nil {
return "", err
}
defer control.Close()
items, err := control.Inventory(ctx)
if err != nil {
return "", err
}
parts := make([]string, 0, len(items))
for _, item := range items {
parts = append(parts, fmt.Sprintf("%s=%s running=%t healthy=%t", m.serviceOf(item.ID), item.ID, item.Running, item.Healthy))
}
sort.Strings(parts)
return strings.Join(parts, "; "), nil
}
func (m *Manager) serviceOf(id string) string {
return "container:" + id
}
// VerifyRunning 确认目标入口真的在服务(部署与恢复后都要看,不看容器状态)。
func (m *Manager) VerifyRunning(ctx context.Context) error {
return m.ingressReady(ctx)
}
// ingressClosed:入口关掉的证据是"连不上",不是"返回错误页"——
// 502/503 说明还有一层代理在对外服务,那不是维护窗口。
func (m *Manager) ingressClosed(ctx context.Context) error {
deadline := time.Now().Add(m.timeout)
var last error
for {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.target.Ingress, nil)
if err != nil {
return err
}
// Maintenance probes must not leave keep-alive connections in the TCP
// ingress: the probe itself would otherwise prevent draining.
req.Close = true
resp, err := m.client.Do(req)
if err != nil {
return nil
}
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
resp.Body.Close()
last = fmt.Errorf("入口仍返回 HTTP %d", resp.StatusCode)
if time.Now().After(deadline) {
return last
}
select {
case <-ctx.Done():
return last
case <-time.After(200 * time.Millisecond):
}
}
}
// ingressReady:入口回来 = 真的能返回 200,而不是"容器起来了"。
func (m *Manager) ingressReady(ctx context.Context) error {
deadline := time.Now().Add(m.timeout)
var last error
for {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.target.Ingress, nil)
if err != nil {
return err
}
// Maintenance probes must not leave keep-alive connections in the TCP
// ingress: the probe itself would otherwise prevent draining.
req.Close = true
resp, err := m.client.Do(req)
if err == nil {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return nil
}
last = fmt.Errorf("入口返回 HTTP %d:%s", resp.StatusCode, strings.TrimSpace(string(body)))
} else {
last = err
}
if time.Now().After(deadline) {
return fmt.Errorf("目标入口没有恢复:%w", last)
}
select {
case <-ctx.Done():
return last
case <-time.After(500 * time.Millisecond):
}
}
}
// DatabaseIdentity 把连接串收敛成"host:port/db"这种可以写进维护记录的身份:
// 维护记录里不落库凭据,但必须能说清动的是哪个库。
func DatabaseIdentity(raw string) (string, error) {
parsed, err := url.Parse(strings.TrimSpace(raw))
if err != nil || parsed.Host == "" {
return "", errors.New("目标数据库连接串无效")
}
database := strings.TrimPrefix(parsed.Path, "/")
if database == "" {
return "", errors.New("目标数据库连接串缺少库名")
}
return parsed.Host + "/" + database, nil
}
const composeServiceLabel = "com.docker.compose.service"
// VerifyTopology 强制"入口唯一":
// - 写入者不许发布任何宿主机端口(否则就存在绕闸直连,维护窗口覆盖不到);
// - 入口闸门必须存在、必须是独立服务,并且发布了入口 URL 用的那个端口。
//
// 任何一条不满足都拒绝开窗口——拓扑错了,"维护"就只是关了一半的门。
func (m *Manager) VerifyTopology(ctx context.Context) error {
items, err := m.api.listProject(ctx, m.target.Project)
if err != nil {
return err
}
byService := make(map[string]containerSummary, len(items))
for _, item := range items {
name := item.Labels[composeServiceLabel]
if name == "" {
return errors.New("项目里有容器缺少 compose service 标签,无法确认写入者边界")
}
if _, duplicated := byService[name]; duplicated {
return fmt.Errorf("项目里服务 %s 有多个实例,无法确认写入者边界", name)
}
byService[name] = item
}
for _, writer := range m.writers {
item, ok := byService[writer]
if !ok {
return fmt.Errorf("声明的写入者 %s 不存在", writer)
}
state, err := m.api.inspect(ctx, item.ID)
if err != nil {
return err
}
if ports := state.publishedPorts(); len(ports) > 0 {
return fmt.Errorf("写入者 %s 直接发布了端口 %s:存在绕开入口闸门的直连通道,拒绝开维护窗口",
writer, strings.Join(ports, ","))
}
}
if m.gate == nil {
return errors.New("实例级维护必须声明入口闸门")
}
for _, writer := range m.writers {
if writer == m.gate.service {
return errors.New("入口闸门不能同时被声明为写入者")
}
}
gateItem, ok := byService[m.gate.service]
if !ok {
return fmt.Errorf("声明的入口闸门 %s 不存在", m.gate.service)
}
gateState, err := m.api.inspect(ctx, gateItem.ID)
if err != nil {
return err
}
published := gateState.publishedPorts()
if len(published) == 0 {
return fmt.Errorf("入口闸门 %s 没有发布任何端口:外部无法访问,入口定义不成立", m.gate.service)
}
// 入口 URL 的端口必须真的是这个闸门发布的端口。
parsed, err := url.Parse(m.target.Ingress)
if err != nil || parsed.Host == "" {
return errors.New("目标入口地址无效")
}
hostPort := parsed.Port()
if hostPort == "" {
if parsed.Scheme == "https" {
hostPort = "443"
} else {
hostPort = "80"
}
}
for _, entry := range published {
if strings.Contains(entry, ":"+hostPort+"->") {
return nil
}
}
return fmt.Errorf("入口 %s 的端口 %s 不是闸门发布的端口(%s):入口指向了别的服务",
m.target.Ingress, hostPort, strings.Join(published, ","))
}