feat(library): 入口闸门排空 + 维护占用可再接管
build / build (push) Successful in 1m28s

被代理的 WeKnora 收到 SIGTERM 会先关监听再 exit(1),在途请求被切断。
排空改到真正的连接入口:闸门停止接客、等在途连接归零、写 drain.json
并 exit 0;协调器只认这条证据,写者退出码只作记录。

顺带修掉维护占用的一条死锁:释放后的窗口此前不能被新作业接管,
等于第一次同步之后谁也再进不去;持有中的窗口仍只允许原作业重入。
This commit is contained in:
姚章浩
2026-10-28 23:35:15 +08:00
parent 6fe98c9a30
commit 38d04488de
10 changed files with 533 additions and 67 deletions
+7
View File
@@ -16,6 +16,13 @@
- 迁移 `000071_library_maintenance_level`:维护记录新增 `level`(instance/library)、
`target_fingerprint`、`services`、`ingress_state`,页面/审计能看清这次到底动的是实例还是只锁了一个库。
- 入口闸门(`LIBRARY_TARGET_INGRESS_SERVICE` + `INGRESS_DRAIN_FILE`):被代理的 WeKnora 收到 SIGTERM
会先关监听再因 `use of closed network connection` 直接 exit(1),在途请求会被切断。
因此排空放在真正的连接入口上做:闸门先停止接受新连接、等在途连接归零、写 `drain.json`
并 exit 0;协调器只认「闸门 exit 0 + phase=drained」这一条证据,写者的退出码只作为记录。
- 维护占用释放后可被新作业接管(此前 `ON CONFLICT ... WHERE owner_job_id = ...` 会让
"第一次同步之后谁也再进不去");仍在持有的窗口依然只允许原作业重入。
### Fixed
- 进程被杀/授权过期不再误放行半写目标:journal 里的 `content_risk` 一旦置位,
+11 -9
View File
@@ -73,15 +73,17 @@ func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manage
return nil, err
}
return librarymaintenance.New(librarymaintenance.Config{
Instance: config.LibraryTargetInstance,
Project: config.LibraryTargetProject,
Writers: config.LibraryTargetWriters,
NonWriters: config.LibraryTargetNonWriters,
DockerSocket: config.LibraryTargetDockerSocket,
IngressURL: config.LibraryTargetIngressURL,
Database: database,
Index: config.LibraryProductionIndex,
Bucket: config.LibraryProductionS3Bucket,
Instance: config.LibraryTargetInstance,
Project: config.LibraryTargetProject,
Writers: config.LibraryTargetWriters,
NonWriters: config.LibraryTargetNonWriters,
DockerSocket: config.LibraryTargetDockerSocket,
IngressURL: config.LibraryTargetIngressURL,
IngressService: config.LibraryTargetIngressService,
IngressDrainFile: config.LibraryTargetIngressDrainFile,
Database: database,
Index: config.LibraryProductionIndex,
Bucket: config.LibraryProductionS3Bucket,
})
}
+51 -37
View File
@@ -102,7 +102,10 @@ type Config struct {
LibraryTargetNonWriters map[string]string
LibraryTargetDockerSocket string
LibraryTargetIngressURL string
LibraryTargetJournalRoot string
// 入口闸门:关入口与排空在它身上做(被代理服务自己不排空)。
LibraryTargetIngressService string
LibraryTargetIngressDrainFile string
LibraryTargetJournalRoot string
// LibraryAgentSocket 是主机侧封存 agent 的 unix socket(只由 worker 连接)。
// 留空即"未接线":页面不显示可用的封存入口,接口返回原因而不是假装已排队。
LibraryAgentSocket string
@@ -170,42 +173,44 @@ func Load(getenv func(string) string) (Config, error) {
TestAcceptancePolicyVersion: valueOrDefault(getenv("TEST_ACCEPTANCE_POLICY_VERSION"), "structural-v1"),
TestAutoReviewPolicy: valueOrDefault(getenv("TEST_AUTO_REVIEW_POLICY"), string(governance.AutoReviewInternal)),
// 整库逻辑备份的目标主机:三项同时给出才接线(脚本目录/备份根有默认值)。
BackupHostSSHConfig: strings.TrimSpace(getenv("BACKUP_HOST_SSH_CONFIG")),
BackupHostTarget: strings.TrimSpace(getenv("BACKUP_HOST_TARGET")),
BackupHostScriptDir: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_SCRIPT_DIR")), "/opt/weknora-test"),
BackupHostRoot: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_ROOT")), "/srv/kb-backups"),
VersionHistoryGiteaBaseURL: strings.TrimSpace(getenv("VERSION_HISTORY_GITEA_BASE_URL")),
VersionHistoryRepo: valueOrDefault(strings.TrimSpace(getenv("VERSION_HISTORY_REPO")), "Weknora/knowledge-governance"),
VersionHistoryAllowedOrigins: splitOrigins(getenv("VERSION_HISTORY_ALLOWED_ORIGINS")),
LibraryArchiveRoot: strings.TrimSpace(getenv("LIBRARY_ARCHIVE_ROOT")),
LibraryAgentSocket: strings.TrimSpace(getenv("LIBRARY_AGENT_SOCKET")),
LibraryProductionDatabaseURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_DATABASE_URL")),
LibraryProductionElasticURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_ELASTIC_URL")),
LibraryProductionIndex: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_INDEX")), "weknora_full_chunks"),
LibraryProductionTenantID: uint64(tenantID),
LibraryProductionInternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_INTERNAL")),
LibraryProductionExternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_EXTERNAL")),
LibraryProductionS3Provider: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_PROVIDER")), "s3"),
LibraryProductionS3Endpoint: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ENDPOINT")),
LibraryProductionS3Region: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_REGION")), "us-east-1"),
LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")),
LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")),
LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")),
LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")),
LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")),
LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")),
LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")),
LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"),
LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")),
LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")),
LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"),
LibraryEmbeddingDimension: embeddingDimension,
LibraryCapacityBudgetBytes: capacityBudget,
LibraryRetentionVersions: retentionVersions,
LibraryRetentionDays: retentionDays,
VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")),
VersionHistoryHTTPTimeout: 15 * time.Second,
VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200),
BackupHostSSHConfig: strings.TrimSpace(getenv("BACKUP_HOST_SSH_CONFIG")),
BackupHostTarget: strings.TrimSpace(getenv("BACKUP_HOST_TARGET")),
BackupHostScriptDir: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_SCRIPT_DIR")), "/opt/weknora-test"),
BackupHostRoot: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_ROOT")), "/srv/kb-backups"),
VersionHistoryGiteaBaseURL: strings.TrimSpace(getenv("VERSION_HISTORY_GITEA_BASE_URL")),
VersionHistoryRepo: valueOrDefault(strings.TrimSpace(getenv("VERSION_HISTORY_REPO")), "Weknora/knowledge-governance"),
VersionHistoryAllowedOrigins: splitOrigins(getenv("VERSION_HISTORY_ALLOWED_ORIGINS")),
LibraryArchiveRoot: strings.TrimSpace(getenv("LIBRARY_ARCHIVE_ROOT")),
LibraryAgentSocket: strings.TrimSpace(getenv("LIBRARY_AGENT_SOCKET")),
LibraryProductionDatabaseURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_DATABASE_URL")),
LibraryProductionElasticURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_ELASTIC_URL")),
LibraryProductionIndex: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_INDEX")), "weknora_full_chunks"),
LibraryProductionTenantID: uint64(tenantID),
LibraryProductionInternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_INTERNAL")),
LibraryProductionExternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_EXTERNAL")),
LibraryProductionS3Provider: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_PROVIDER")), "s3"),
LibraryProductionS3Endpoint: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ENDPOINT")),
LibraryProductionS3Region: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_REGION")), "us-east-1"),
LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")),
LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")),
LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")),
LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")),
LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")),
LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")),
LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")),
LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"),
LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")),
LibraryTargetIngressService: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_SERVICE")),
LibraryTargetIngressDrainFile: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_DRAIN_FILE")),
LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")),
LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"),
LibraryEmbeddingDimension: embeddingDimension,
LibraryCapacityBudgetBytes: capacityBudget,
LibraryRetentionVersions: retentionVersions,
LibraryRetentionDays: retentionDays,
VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")),
VersionHistoryHTTPTimeout: 15 * time.Second,
VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200),
}
if config.DatabaseURL == "" {
return Config{}, errors.New("DATABASE_URL is required")
@@ -338,6 +343,12 @@ func Load(getenv func(string) string) (Config, error) {
if config.LibraryTargetJournalRoot == "" {
missing = append(missing, "LIBRARY_TARGET_JOURNAL_ROOT")
}
if config.LibraryTargetIngressService == "" {
missing = append(missing, "LIBRARY_TARGET_INGRESS_SERVICE")
}
if config.LibraryTargetIngressDrainFile == "" {
missing = append(missing, "LIBRARY_TARGET_INGRESS_DRAIN_FILE")
}
if len(missing) > 0 {
return Config{}, fmt.Errorf("目标维护配置不完整:缺少 %s", strings.Join(missing, "、"))
}
@@ -347,6 +358,9 @@ func Load(getenv func(string) string) (Config, error) {
if !filepath.IsAbs(config.LibraryTargetDockerSocket) {
return Config{}, errors.New("LIBRARY_TARGET_DOCKER_SOCKET 必须是绝对路径")
}
if !filepath.IsAbs(config.LibraryTargetIngressDrainFile) {
return Config{}, errors.New("LIBRARY_TARGET_INGRESS_DRAIN_FILE 必须是绝对路径")
}
if len(config.LibraryTargetNonWriters) == 0 {
return Config{}, errors.New("LIBRARY_TARGET_NON_WRITERS 必须逐项说明项目内非写入服务")
}
+62 -10
View File
@@ -29,6 +29,24 @@ type Service struct {
ExitCode *int `json:"exit_code,omitempty"`
}
// GateEvidence 是"入口真的关了、在途真的排空了"的证据。
// 只有被代理的服务自己不会优雅退出时,才知道为什么要在入口这一层排空。
type GateEvidence struct {
Closed bool `json:"closed"`
Drained bool `json:"drained"`
ActiveAtClose int `json:"active_at_close"`
Served int `json:"served"`
Detail string `json:"detail,omitempty"`
}
// Gate 是目标的唯一对外入口:维护窗口先关它、等在途请求结束,再停写入者。
type Gate interface {
// CloseIngress 必须等到在途请求结束才返回成功;返回的不是"已发信号"而是"已排空"。
CloseIngress(context.Context) (GateEvidence, error)
// OpenIngress 恢复入口,并确认它真的在服务。
OpenIngress(context.Context) error
}
// Inventory must enumerate the complete, independently scoped test writer set,
// using immutable instance IDs. Never implement it as only the currently running
// subset. Start must wait for readiness; callers still read back the result.
@@ -54,16 +72,21 @@ type journal struct {
Result
// Level 区分"实例级维护"(写入者/入口被真的关掉)与"仅治理库互斥"。
// 没有 Level 的旧 journal 来自封存链路,按实例级处理但不冒充已验证。
Level string `json:"level,omitempty"`
Ingress string `json:"ingress,omitempty"`
IngressClosed bool `json:"ingress_closed,omitempty"`
DrainVerified bool `json:"drain_verified,omitempty"`
ContentRisk bool `json:"content_risk,omitempty"`
RestoreBlocked bool `json:"restore_blocked,omitempty"`
Services []serviceRecord `json:"services"`
StartedAt time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
Code string `json:"code,omitempty"`
Level string `json:"level,omitempty"`
Ingress string `json:"ingress,omitempty"`
IngressClosed bool `json:"ingress_closed,omitempty"`
DrainVerified bool `json:"drain_verified,omitempty"`
ContentRisk bool `json:"content_risk,omitempty"`
RestoreBlocked bool `json:"restore_blocked,omitempty"`
// Gate 记录入口关闭/排空的证据;没有它就不能说"入口关了"。
// 入口恢复后证据仍保留(GateReopened=true),否则事后查不到
// "关的时候还有几个在途请求"。
Gate *GateEvidence `json:"gate,omitempty"`
GateReopened bool `json:"gate_reopened,omitempty"`
Services []serviceRecord `json:"services"`
StartedAt time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
Code string `json:"code,omitempty"`
}
type Coordinator struct {
root *os.Root
@@ -75,6 +98,7 @@ type Coordinator struct {
// 证明入口真的关了、真的回来了,而不是"停了容器就算关"。
ingressClosed func(context.Context) error
ingressReady func(context.Context) error
gate Gate
}
func Open(root, source string, c Controller) (*Coordinator, error) {
@@ -116,6 +140,15 @@ func (c *Coordinator) SetTarget(level, ingress string, closed, ready func(contex
return nil
}
// SetGate 声明目标入口闸门:实例级维护里"关入口"就是关它、并从它拿到排空证据。
func (c *Coordinator) SetGate(g Gate) error {
if g == nil {
return errors.New("invalid ingress gate")
}
c.gate = g
return nil
}
func validIngress(raw string) bool {
parsed, err := url.Parse(raw)
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.Path != "" && parsed.User == nil
@@ -312,6 +345,12 @@ func (c *Coordinator) load(job string) (journal, error) {
if j.RestoreBlocked && !j.ContentRisk {
return journal{}, errors.New("invalid freeze restore block")
}
if j.Gate != nil && (!j.Gate.Closed || j.Gate.ActiveAtClose < 0 || j.Gate.Served < 0) {
return journal{}, errors.New("invalid freeze ingress evidence")
}
if j.GateReopened && j.Gate == nil {
return journal{}, errors.New("ingress reopened without close evidence")
}
if len(j.Services) == 0 || len(j.Services) > 256 {
return journal{}, errors.New("invalid freeze journal services")
}
@@ -444,5 +483,18 @@ func (c *Coordinator) restore(ctx context.Context, j *journal) error {
failures = errors.Join(failures, errors.New("original test service missing; replacement not started"))
}
}
// 服务都回来了才开放入口:先开门再启动服务,等于把流量打向没准备好的目标。
if j.Gate != nil {
if c.gate == nil {
failures = errors.Join(failures, errors.New("入口闸门缺失:不能确认目标已恢复"))
} else if err := c.gate.OpenIngress(ctx); err != nil {
failures = errors.Join(failures, fmt.Errorf("目标入口恢复失败:%w", err))
} else {
j.GateReopened = true
if err := c.save(*j); err != nil {
failures = errors.Join(failures, err)
}
}
}
return failures
}
+24 -4
View File
@@ -125,7 +125,23 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co
}
}
// 停写入者 + 排空 + 关入口:任何一步不成立都不许往下写。
// 顺序是有原因的:入口先停(新请求被拒、在途请求跑完),写者后停。
// 反过来做,正在处理的请求就会被 SIGTERM 切断——那不算排空。
if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) {
evidence, gateErr := c.gate.CloseIngress(ctx)
if gateErr != nil {
return c.abandonWindow(ctx, &j, fmt.Errorf("关闭目标入口失败:%w", gateErr), "gate_failed")
}
if !evidence.Closed || !evidence.Drained {
return c.abandonWindow(ctx, &j, errors.New("目标入口未确认排空(仍有在途请求)"), "gate_not_drained")
}
j.Gate = &evidence
if err = c.save(j); err != nil {
return Result{}, err
}
}
// 停写入者:任何一步不成立都不许往下写。
if stopErr := c.stopAll(ctx, &j); stopErr != nil {
return c.abandonWindow(ctx, &j, stopErr, "stop_failed")
}
@@ -284,6 +300,10 @@ func (c *Coordinator) verifyDrain(ctx context.Context, j *journal) error {
if needed == 0 {
return nil
}
// 入口层已经证明"在途请求全部结束"时,写者的退出码只作为记录:
// 被代理的 WeKnora 收到 SIGTERM 会自己 exit(1)(upstream bug),
// 那不代表请求被切断——请求早在入口关闭时就已经跑完了。
gateDrained := j.Gate != nil && j.Gate.Closed && j.Gate.Drained
items, err := inventory(ctx, c.control)
if err != nil {
return err
@@ -303,18 +323,18 @@ func (c *Coordinator) verifyDrain(ctx context.Context, j *journal) error {
return errors.New("test writer still running")
}
if now.ExitCode == nil {
if c.level == "instance" {
if c.level == "instance" && !gateDrained {
return fmt.Errorf("无法证明 %s 的在途请求已排空(控制器没有退出码)", s.ID)
}
continue
}
s.ExitCode = now.ExitCode
if *now.ExitCode != 0 {
if *now.ExitCode != 0 && !gateDrained {
return fmt.Errorf("写入者 %s 未优雅退出(exit=%d):在途请求没有完成排空", s.ID, *now.ExitCode)
}
verified++
}
j.DrainVerified = verified == needed
j.DrainVerified = gateDrained || verified == needed
return nil
}
+61
View File
@@ -251,3 +251,64 @@ func TestLibraryLevelRejectsIngressClaim(t *testing.T) {
t.Fatal("empty level accepted")
}
}
// fakeGate 记录"入口先关、写者后停、恢复时先起服务再开入口"的顺序。
type fakeGate struct {
calls *[]string
drain bool
active int
}
func (g *fakeGate) CloseIngress(context.Context) (GateEvidence, error) {
*g.calls = append(*g.calls, "gate:close")
return GateEvidence{Closed: true, Drained: g.drain, ActiveAtClose: g.active, Served: 3}, nil
}
func (g *fakeGate) OpenIngress(context.Context) error {
*g.calls = append(*g.calls, "gate:open")
return nil
}
func TestWindowClosesGateBeforeStoppingWriters(t *testing.T) {
_, c, e := instanceFixture(t)
e.ingressClosed = func(context.Context) error { return nil }
gate := &fakeGate{calls: &c.calls, drain: true, active: 1}
if err := e.SetGate(gate); err != nil {
t.Fatal(err)
}
_, err := e.RunWindow(context.Background(), "job-gate", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true })
if err != nil {
t.Fatal(err)
}
want := "gate:close,stop:app,stop:store,start:app,start:store,gate:open"
if strings.Join(c.calls, ",") != want {
t.Fatalf("calls=%v want=%s", c.calls, want)
}
// 写者退出码非 0 也不影响结论:排空是在入口层证明的(被代理服务自己不优雅退出)。
j, err := e.load("job-gate")
if err != nil || !j.DrainVerified {
t.Fatalf("journal=%+v err=%v", j, err)
}
// 入口恢复后证据仍留着:报告要能说清"关的时候还有几个在途请求"。
if j.Gate == nil || !j.GateReopened {
t.Fatalf("gate evidence lost after reopening: %+v reopened=%v", j.Gate, j.GateReopened)
}
}
// 入口没排空(还有在途请求)时不许停写者、更不许写目标。
func TestWindowRefusesWhenGateNotDrained(t *testing.T) {
_, c, e := instanceFixture(t)
gate := &fakeGate{calls: &c.calls, drain: false, active: 2}
if err := e.SetGate(gate); err != nil {
t.Fatal(err)
}
work := 0
_, err := e.RunWindow(context.Background(), "job-gate-bad", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || work != 0 {
t.Fatalf("err=%v work=%d", err, work)
}
if strings.Join(c.calls, ",") != "gate:close" {
t.Fatalf("writers were stopped before the ingress was drained: %v", c.calls)
}
}
+214
View File
@@ -0,0 +1,214 @@
package librarymaintenance
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"time"
"knowledge-governance/internal/libraryfreeze"
)
// dockerGate 是目标入口闸门的真实实现:它只认"容器已经优雅退出"+
// 闸门自己写下的 drain.json(停止接客时还有多少在途连接、最后是否归零)。
// 不看 HTTP 探测就宣布"入口关了"——那是猜。
type dockerGate struct {
client *http.Client
socket string
project string
service string
drainFile string
timeout time.Duration
}
func newDockerGate(socket, project, service, drainFile string, timeout time.Duration) (*dockerGate, error) {
if !filepath.IsAbs(socket) || project == "" || service == "" || !filepath.IsAbs(drainFile) {
return nil, errors.New("入口闸门配置不完整")
}
info, err := os.Stat(socket)
if err != nil || info.Mode()&os.ModeSocket == 0 {
return nil, errors.New("local Docker socket unavailable")
}
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", socket)
}}
return &dockerGate{
client: &http.Client{Transport: transport, Timeout: timeout,
CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("docker redirects prohibited") }},
socket: socket, project: project, service: service, drainFile: drainFile, timeout: timeout,
}, nil
}
func (g *dockerGate) request(ctx context.Context, method, path string, out any) error {
req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil)
if err != nil {
return errors.New("invalid docker request")
}
resp, err := g.client.Do(req)
if err != nil {
return errors.New("docker request failed")
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("docker request rejected: %d", resp.StatusCode)
}
if out != nil {
data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if readErr != nil || json.Unmarshal(data, out) != nil {
return errors.New("invalid docker response")
}
}
return nil
}
type gateInspection struct {
ID string `json:"Id"`
Config struct {
Labels map[string]string `json:"Labels"`
} `json:"Config"`
State struct {
Running bool
Status string
ExitCode int
Health *struct{ Status string }
} `json:"State"`
}
func (g *dockerGate) container(ctx context.Context) (gateInspection, error) {
filters, _ := json.Marshal(map[string][]string{"label": {
"com.docker.compose.project=" + g.project,
"com.docker.compose.service=" + g.service,
}})
var listed []struct {
ID string `json:"Id"`
}
if err := g.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &listed); err != nil {
return gateInspection{}, err
}
if len(listed) != 1 {
return gateInspection{}, fmt.Errorf("入口闸门实例数异常:%d", len(listed))
}
var data gateInspection
if err := g.request(ctx, "GET", "/containers/"+listed[0].ID+"/json", &data); err != nil {
return gateInspection{}, err
}
if data.Config.Labels["com.docker.compose.project"] != g.project ||
data.Config.Labels["com.docker.compose.service"] != g.service {
return gateInspection{}, errors.New("入口闸门身份不匹配")
}
return data, nil
}
type drainRecord struct {
Phase string `json:"phase"`
Active int `json:"active"`
Served int `json:"served"`
WaitedMS int64 `json:"waited_ms"`
}
func (g *dockerGate) readDrain() (drainRecord, error) {
file, err := os.Open(g.drainFile)
if err != nil {
return drainRecord{}, err
}
defer file.Close()
var record drainRecord
decoder := json.NewDecoder(io.LimitReader(file, 64<<10))
if err := decoder.Decode(&record); err != nil {
return drainRecord{}, err
}
return record, nil
}
// CloseIngress:先删掉上一次的 drain 记录(不能用旧证据冒充这次),再发 SIGTERM,
// 等到容器自己退出,最后只认 exit 0 + phase=drained。
func (g *dockerGate) CloseIngress(ctx context.Context) (libraryfreeze.GateEvidence, error) {
bounded, cancel := context.WithTimeout(ctx, g.timeout)
defer cancel()
data, err := g.container(bounded)
if err != nil {
return libraryfreeze.GateEvidence{}, err
}
if data.State.Running {
if err := os.Remove(g.drainFile); err != nil && !errors.Is(err, os.ErrNotExist) {
return libraryfreeze.GateEvidence{}, fmt.Errorf("无法清理上次的排空记录:%w", err)
}
if err := g.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil {
return libraryfreeze.GateEvidence{}, err
}
deadline := time.Now().Add(g.timeout)
for {
state, err := g.container(bounded)
if err != nil {
return libraryfreeze.GateEvidence{}, err
}
if !state.State.Running {
data = state
break
}
if time.Now().After(deadline) {
return libraryfreeze.GateEvidence{}, errors.New("入口闸门没有在期限内退出(在途请求可能还没结束)")
}
select {
case <-bounded.Done():
return libraryfreeze.GateEvidence{}, errors.New("入口闸门退出等待被中断")
case <-time.After(100 * time.Millisecond):
}
}
}
record, err := g.readDrain()
if err != nil {
return libraryfreeze.GateEvidence{}, fmt.Errorf("读不到入口排空记录:%w", err)
}
if record.Phase != "drained" || data.State.ExitCode != 0 {
return libraryfreeze.GateEvidence{}, fmt.Errorf("入口排空未确认(phase=%s exit=%d active=%d)",
record.Phase, data.State.ExitCode, record.Active)
}
return libraryfreeze.GateEvidence{Closed: true, Drained: true, ActiveAtClose: record.Active,
Served: record.Served, Detail: fmt.Sprintf("gate exit=0 phase=%s waited_ms=%d", record.Phase, record.WaitedMS)}, nil
}
// OpenIngress:把闸门拉起来并等健康检查通过;能不能真的服务由入口探针再确认一次。
func (g *dockerGate) OpenIngress(ctx context.Context) error {
bounded, cancel := context.WithTimeout(ctx, g.timeout)
defer cancel()
data, err := g.container(bounded)
if err != nil {
return err
}
if !data.State.Running {
if err := g.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil {
return err
}
}
deadline := time.Now().Add(g.timeout)
for {
state, err := g.container(bounded)
if err != nil {
return err
}
healthy := state.State.Running && state.State.Health != nil && state.State.Health.Status == "healthy"
if healthy {
// 记下新的 serving 记录:下一次关闭要用它自己的证据。
if _, err := g.readDrain(); err == nil {
return nil
}
return nil
}
if time.Now().After(deadline) {
return errors.New("入口闸门没有在期限内恢复健康")
}
select {
case <-bounded.Done():
return errors.New("入口闸门恢复等待被中断")
case <-time.After(200 * time.Millisecond):
}
}
}
+26 -6
View File
@@ -49,10 +49,14 @@ type Config struct {
NonWriters map[string]string
DockerSocket string
IngressURL string
Database string
Index string
Bucket string
Timeout time.Duration
// IngressService 是入口闸门在 compose 项目里的服务名;DrainFile 是它写
// 排空证据的宿主机文件。两者缺一,就只能说"停了写入者",不能说"排空了"。
IngressService string
IngressDrainFile string
Database string
Index string
Bucket string
Timeout time.Duration
}
type Manager struct {
@@ -63,6 +67,7 @@ type Manager struct {
nonWriters map[string]string
timeout time.Duration
client *http.Client
gate *dockerGate
}
func New(cfg Config) (*Manager, error) {
@@ -97,9 +102,17 @@ func New(cfg Config) (*Manager, error) {
return nil, err
}
target.Fingerprint = fingerprint
return &Manager{target: target, source: project, docker: cfg.DockerSocket,
manager := &Manager{target: target, source: project, docker: cfg.DockerSocket,
writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout,
client: &http.Client{Timeout: 5 * time.Second}}, nil
client: &http.Client{Timeout: 5 * time.Second}}
if cfg.IngressService != "" || cfg.IngressDrainFile != "" {
gate, err := newDockerGate(cfg.DockerSocket, project, cfg.IngressService, cfg.IngressDrainFile, cfg.Timeout)
if err != nil {
return nil, err
}
manager.gate = gate
}
return manager, nil
}
func fingerprintOf(t Target) (string, error) {
@@ -146,6 +159,13 @@ func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.
control.Close()
return nil, nil, err
}
if m.gate != nil {
if err := coordinator.SetGate(m.gate); err != nil {
coordinator.Close()
control.Close()
return nil, nil, err
}
}
return coordinator, control, nil
}
@@ -72,7 +72,10 @@ ON CONFLICT (scope) DO UPDATE SET
note = EXCLUDED.note,
updated_at = EXCLUDED.updated_at,
released_at = NULL
WHERE library_maintenance.owner_job_id = EXCLUDED.owner_job_id
-- 已经释放的窗口允许被新作业接管(否则第一个同步之后谁也再进不去);
-- 仍在持有的窗口只允许原作业重入。
WHERE library_maintenance.released_at IS NOT NULL
OR library_maintenance.owner_job_id = EXCLUDED.owner_job_id
RETURNING `+maintenanceColumns,
item.Scope, item.OwnerJobID, item.TargetIdentity, level, item.TargetFingerprint, string(raw),
ingress, item.State, item.OriginalServiceState, item.Note, now.UTC())
@@ -0,0 +1,73 @@
package postgres
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
// 维护占用的生命周期必须只有一条规则:
// - 正在持有的窗口只允许原作业重入,别的作业一律拒绝;
// - 已经释放的窗口必须能被新作业接管,否则"第一次同步之后谁也再进不去"。
func TestLibraryMaintenanceOwnershipAcrossRelease(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatalf("connect: %v", err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("migrate: %v", err)
}
store := NewStore(pool)
now := time.Now().UTC()
first, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{
Scope: "internal", OwnerJobID: "job-1", TargetIdentity: "instance=kgmaint",
Level: "instance", TargetFingerprint: "fp", Services: []string{"app"},
IngressState: "unknown", State: "acquired", OriginalServiceState: "app=running",
Note: "test",
}, now)
if err != nil {
t.Fatalf("first acquire: %v", err)
}
if first.Level != "instance" || len(first.Services) != 1 || first.Services[0] != "app" {
t.Fatalf("acquired row lost identity: %+v", first)
}
// 同一作业重入(接管)必须成功。
if _, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{
Scope: "internal", OwnerJobID: "job-1", State: "applying", Level: "instance",
}, now.Add(time.Second)); err != nil {
t.Fatalf("same owner re-entry rejected: %v", err)
}
// 另一个作业在持有期间必须被拒。
if _, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{
Scope: "internal", OwnerJobID: "job-2", State: "acquired", Level: "instance",
}, now.Add(time.Second)); !errors.Is(err, ErrMaintenanceHeld) {
t.Fatalf("second owner during hold: err=%v", err)
}
if err := store.ReleaseLibraryMaintenance(ctx, "internal", "job-1", now.Add(2*time.Second)); err != nil {
t.Fatalf("release: %v", err)
}
second, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{
Scope: "internal", OwnerJobID: "job-2", TargetIdentity: "instance=kgmaint",
Level: "instance", TargetFingerprint: "fp2", Services: []string{"app"},
IngressState: "unknown", State: "acquired", Note: "second",
}, now.Add(3*time.Second))
if err != nil {
t.Fatalf("acquire after release: %v", err)
}
if second.OwnerJobID != "job-2" || second.ReleasedAt != nil || second.TargetFingerprint != "fp2" {
t.Fatalf("released window not re-acquired cleanly: %+v", second)
}
}