From 38d04488de00455b29e3d62d3d34de7271d05eaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Wed, 28 Oct 2026 23:35:15 +0800 Subject: [PATCH] =?UTF-8?q?feat(library):=20=E5=85=A5=E5=8F=A3=E9=97=B8?= =?UTF-8?q?=E9=97=A8=E6=8E=92=E7=A9=BA=20+=20=E7=BB=B4=E6=8A=A4=E5=8D=A0?= =?UTF-8?q?=E7=94=A8=E5=8F=AF=E5=86=8D=E6=8E=A5=E7=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 被代理的 WeKnora 收到 SIGTERM 会先关监听再 exit(1),在途请求被切断。 排空改到真正的连接入口:闸门停止接客、等在途连接归零、写 drain.json 并 exit 0;协调器只认这条证据,写者退出码只作记录。 顺带修掉维护占用的一条死锁:释放后的窗口此前不能被新作业接管, 等于第一次同步之后谁也再进不去;持有中的窗口仍只允许原作业重入。 --- CHANGELOG.md | 7 + cmd/worker/library_handlers.go | 20 +- internal/appconfig/config.go | 88 ++++--- internal/libraryfreeze/freeze.go | 72 +++++- internal/libraryfreeze/window.go | 28 ++- internal/libraryfreeze/window_test.go | 61 +++++ internal/librarymaintenance/gate.go | 214 ++++++++++++++++++ internal/librarymaintenance/target.go | 32 ++- .../platform/postgres/library_maintenance.go | 5 +- .../library_maintenance_integration_test.go | 73 ++++++ 10 files changed, 533 insertions(+), 67 deletions(-) create mode 100644 internal/librarymaintenance/gate.go create mode 100644 internal/platform/postgres/library_maintenance_integration_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index bf775f2..26b6a2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,13 @@ - 迁移 `000071_library_maintenance_level`:维护记录新增 `level`(instance/library)、 `target_fingerprint`、`services`、`ingress_state`,页面/审计能看清这次到底动的是实例还是只锁了一个库。 +- 入口闸门(`LIBRARY_TARGET_INGRESS_SERVICE` + `INGRESS_DRAIN_FILE`):被代理的 WeKnora 收到 SIGTERM + 会先关监听再因 `use of closed network connection` 直接 exit(1),在途请求会被切断。 + 因此排空放在真正的连接入口上做:闸门先停止接受新连接、等在途连接归零、写 `drain.json` + 并 exit 0;协调器只认「闸门 exit 0 + phase=drained」这一条证据,写者的退出码只作为记录。 +- 维护占用释放后可被新作业接管(此前 `ON CONFLICT ... WHERE owner_job_id = ...` 会让 + "第一次同步之后谁也再进不去");仍在持有的窗口依然只允许原作业重入。 + ### Fixed - 进程被杀/授权过期不再误放行半写目标:journal 里的 `content_risk` 一旦置位, diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index b32f6fc..154e2cf 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -73,15 +73,17 @@ func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manage return nil, err } return librarymaintenance.New(librarymaintenance.Config{ - Instance: config.LibraryTargetInstance, - Project: config.LibraryTargetProject, - Writers: config.LibraryTargetWriters, - NonWriters: config.LibraryTargetNonWriters, - DockerSocket: config.LibraryTargetDockerSocket, - IngressURL: config.LibraryTargetIngressURL, - Database: database, - Index: config.LibraryProductionIndex, - Bucket: config.LibraryProductionS3Bucket, + Instance: config.LibraryTargetInstance, + Project: config.LibraryTargetProject, + Writers: config.LibraryTargetWriters, + NonWriters: config.LibraryTargetNonWriters, + DockerSocket: config.LibraryTargetDockerSocket, + IngressURL: config.LibraryTargetIngressURL, + IngressService: config.LibraryTargetIngressService, + IngressDrainFile: config.LibraryTargetIngressDrainFile, + Database: database, + Index: config.LibraryProductionIndex, + Bucket: config.LibraryProductionS3Bucket, }) } diff --git a/internal/appconfig/config.go b/internal/appconfig/config.go index 7d46ae7..f11f362 100644 --- a/internal/appconfig/config.go +++ b/internal/appconfig/config.go @@ -102,7 +102,10 @@ type Config struct { LibraryTargetNonWriters map[string]string LibraryTargetDockerSocket string LibraryTargetIngressURL string - LibraryTargetJournalRoot string + // 入口闸门:关入口与排空在它身上做(被代理服务自己不排空)。 + LibraryTargetIngressService string + LibraryTargetIngressDrainFile string + LibraryTargetJournalRoot string // LibraryAgentSocket 是主机侧封存 agent 的 unix socket(只由 worker 连接)。 // 留空即"未接线":页面不显示可用的封存入口,接口返回原因而不是假装已排队。 LibraryAgentSocket string @@ -170,42 +173,44 @@ func Load(getenv func(string) string) (Config, error) { TestAcceptancePolicyVersion: valueOrDefault(getenv("TEST_ACCEPTANCE_POLICY_VERSION"), "structural-v1"), TestAutoReviewPolicy: valueOrDefault(getenv("TEST_AUTO_REVIEW_POLICY"), string(governance.AutoReviewInternal)), // 整库逻辑备份的目标主机:三项同时给出才接线(脚本目录/备份根有默认值)。 - BackupHostSSHConfig: strings.TrimSpace(getenv("BACKUP_HOST_SSH_CONFIG")), - BackupHostTarget: strings.TrimSpace(getenv("BACKUP_HOST_TARGET")), - BackupHostScriptDir: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_SCRIPT_DIR")), "/opt/weknora-test"), - BackupHostRoot: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_ROOT")), "/srv/kb-backups"), - VersionHistoryGiteaBaseURL: strings.TrimSpace(getenv("VERSION_HISTORY_GITEA_BASE_URL")), - VersionHistoryRepo: valueOrDefault(strings.TrimSpace(getenv("VERSION_HISTORY_REPO")), "Weknora/knowledge-governance"), - VersionHistoryAllowedOrigins: splitOrigins(getenv("VERSION_HISTORY_ALLOWED_ORIGINS")), - LibraryArchiveRoot: strings.TrimSpace(getenv("LIBRARY_ARCHIVE_ROOT")), - LibraryAgentSocket: strings.TrimSpace(getenv("LIBRARY_AGENT_SOCKET")), - LibraryProductionDatabaseURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_DATABASE_URL")), - LibraryProductionElasticURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_ELASTIC_URL")), - LibraryProductionIndex: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_INDEX")), "weknora_full_chunks"), - LibraryProductionTenantID: uint64(tenantID), - LibraryProductionInternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_INTERNAL")), - LibraryProductionExternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_EXTERNAL")), - LibraryProductionS3Provider: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_PROVIDER")), "s3"), - LibraryProductionS3Endpoint: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ENDPOINT")), - LibraryProductionS3Region: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_REGION")), "us-east-1"), - LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")), - LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")), - LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")), - LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")), - LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")), - LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")), - LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")), - LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"), - LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")), - LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")), - LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"), - LibraryEmbeddingDimension: embeddingDimension, - LibraryCapacityBudgetBytes: capacityBudget, - LibraryRetentionVersions: retentionVersions, - LibraryRetentionDays: retentionDays, - VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")), - VersionHistoryHTTPTimeout: 15 * time.Second, - VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200), + BackupHostSSHConfig: strings.TrimSpace(getenv("BACKUP_HOST_SSH_CONFIG")), + BackupHostTarget: strings.TrimSpace(getenv("BACKUP_HOST_TARGET")), + BackupHostScriptDir: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_SCRIPT_DIR")), "/opt/weknora-test"), + BackupHostRoot: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_ROOT")), "/srv/kb-backups"), + VersionHistoryGiteaBaseURL: strings.TrimSpace(getenv("VERSION_HISTORY_GITEA_BASE_URL")), + VersionHistoryRepo: valueOrDefault(strings.TrimSpace(getenv("VERSION_HISTORY_REPO")), "Weknora/knowledge-governance"), + VersionHistoryAllowedOrigins: splitOrigins(getenv("VERSION_HISTORY_ALLOWED_ORIGINS")), + LibraryArchiveRoot: strings.TrimSpace(getenv("LIBRARY_ARCHIVE_ROOT")), + LibraryAgentSocket: strings.TrimSpace(getenv("LIBRARY_AGENT_SOCKET")), + LibraryProductionDatabaseURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_DATABASE_URL")), + LibraryProductionElasticURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_ELASTIC_URL")), + LibraryProductionIndex: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_INDEX")), "weknora_full_chunks"), + LibraryProductionTenantID: uint64(tenantID), + LibraryProductionInternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_INTERNAL")), + LibraryProductionExternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_EXTERNAL")), + LibraryProductionS3Provider: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_PROVIDER")), "s3"), + LibraryProductionS3Endpoint: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ENDPOINT")), + LibraryProductionS3Region: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_REGION")), "us-east-1"), + LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")), + LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")), + LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")), + LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")), + LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")), + LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")), + LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")), + LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"), + LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")), + LibraryTargetIngressService: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_SERVICE")), + LibraryTargetIngressDrainFile: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_DRAIN_FILE")), + LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")), + LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"), + LibraryEmbeddingDimension: embeddingDimension, + LibraryCapacityBudgetBytes: capacityBudget, + LibraryRetentionVersions: retentionVersions, + LibraryRetentionDays: retentionDays, + VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")), + VersionHistoryHTTPTimeout: 15 * time.Second, + VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200), } if config.DatabaseURL == "" { return Config{}, errors.New("DATABASE_URL is required") @@ -338,6 +343,12 @@ func Load(getenv func(string) string) (Config, error) { if config.LibraryTargetJournalRoot == "" { missing = append(missing, "LIBRARY_TARGET_JOURNAL_ROOT") } + if config.LibraryTargetIngressService == "" { + missing = append(missing, "LIBRARY_TARGET_INGRESS_SERVICE") + } + if config.LibraryTargetIngressDrainFile == "" { + missing = append(missing, "LIBRARY_TARGET_INGRESS_DRAIN_FILE") + } if len(missing) > 0 { return Config{}, fmt.Errorf("目标维护配置不完整:缺少 %s", strings.Join(missing, "、")) } @@ -347,6 +358,9 @@ func Load(getenv func(string) string) (Config, error) { if !filepath.IsAbs(config.LibraryTargetDockerSocket) { return Config{}, errors.New("LIBRARY_TARGET_DOCKER_SOCKET 必须是绝对路径") } + if !filepath.IsAbs(config.LibraryTargetIngressDrainFile) { + return Config{}, errors.New("LIBRARY_TARGET_INGRESS_DRAIN_FILE 必须是绝对路径") + } if len(config.LibraryTargetNonWriters) == 0 { return Config{}, errors.New("LIBRARY_TARGET_NON_WRITERS 必须逐项说明项目内非写入服务") } diff --git a/internal/libraryfreeze/freeze.go b/internal/libraryfreeze/freeze.go index 27a9df1..1b3e232 100644 --- a/internal/libraryfreeze/freeze.go +++ b/internal/libraryfreeze/freeze.go @@ -29,6 +29,24 @@ type Service struct { ExitCode *int `json:"exit_code,omitempty"` } +// GateEvidence 是"入口真的关了、在途真的排空了"的证据。 +// 只有被代理的服务自己不会优雅退出时,才知道为什么要在入口这一层排空。 +type GateEvidence struct { + Closed bool `json:"closed"` + Drained bool `json:"drained"` + ActiveAtClose int `json:"active_at_close"` + Served int `json:"served"` + Detail string `json:"detail,omitempty"` +} + +// Gate 是目标的唯一对外入口:维护窗口先关它、等在途请求结束,再停写入者。 +type Gate interface { + // CloseIngress 必须等到在途请求结束才返回成功;返回的不是"已发信号"而是"已排空"。 + CloseIngress(context.Context) (GateEvidence, error) + // OpenIngress 恢复入口,并确认它真的在服务。 + OpenIngress(context.Context) error +} + // Inventory must enumerate the complete, independently scoped test writer set, // using immutable instance IDs. Never implement it as only the currently running // subset. Start must wait for readiness; callers still read back the result. @@ -54,16 +72,21 @@ type journal struct { Result // Level 区分"实例级维护"(写入者/入口被真的关掉)与"仅治理库互斥"。 // 没有 Level 的旧 journal 来自封存链路,按实例级处理但不冒充已验证。 - Level string `json:"level,omitempty"` - Ingress string `json:"ingress,omitempty"` - IngressClosed bool `json:"ingress_closed,omitempty"` - DrainVerified bool `json:"drain_verified,omitempty"` - ContentRisk bool `json:"content_risk,omitempty"` - RestoreBlocked bool `json:"restore_blocked,omitempty"` - Services []serviceRecord `json:"services"` - StartedAt time.Time `json:"started_at"` - FinishedAt *time.Time `json:"finished_at,omitempty"` - Code string `json:"code,omitempty"` + Level string `json:"level,omitempty"` + Ingress string `json:"ingress,omitempty"` + IngressClosed bool `json:"ingress_closed,omitempty"` + DrainVerified bool `json:"drain_verified,omitempty"` + ContentRisk bool `json:"content_risk,omitempty"` + RestoreBlocked bool `json:"restore_blocked,omitempty"` + // Gate 记录入口关闭/排空的证据;没有它就不能说"入口关了"。 + // 入口恢复后证据仍保留(GateReopened=true),否则事后查不到 + // "关的时候还有几个在途请求"。 + Gate *GateEvidence `json:"gate,omitempty"` + GateReopened bool `json:"gate_reopened,omitempty"` + Services []serviceRecord `json:"services"` + StartedAt time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Code string `json:"code,omitempty"` } type Coordinator struct { root *os.Root @@ -75,6 +98,7 @@ type Coordinator struct { // 证明入口真的关了、真的回来了,而不是"停了容器就算关"。 ingressClosed func(context.Context) error ingressReady func(context.Context) error + gate Gate } func Open(root, source string, c Controller) (*Coordinator, error) { @@ -116,6 +140,15 @@ func (c *Coordinator) SetTarget(level, ingress string, closed, ready func(contex return nil } +// SetGate 声明目标入口闸门:实例级维护里"关入口"就是关它、并从它拿到排空证据。 +func (c *Coordinator) SetGate(g Gate) error { + if g == nil { + return errors.New("invalid ingress gate") + } + c.gate = g + return nil +} + func validIngress(raw string) bool { parsed, err := url.Parse(raw) return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.Path != "" && parsed.User == nil @@ -312,6 +345,12 @@ func (c *Coordinator) load(job string) (journal, error) { if j.RestoreBlocked && !j.ContentRisk { return journal{}, errors.New("invalid freeze restore block") } + if j.Gate != nil && (!j.Gate.Closed || j.Gate.ActiveAtClose < 0 || j.Gate.Served < 0) { + return journal{}, errors.New("invalid freeze ingress evidence") + } + if j.GateReopened && j.Gate == nil { + return journal{}, errors.New("ingress reopened without close evidence") + } if len(j.Services) == 0 || len(j.Services) > 256 { return journal{}, errors.New("invalid freeze journal services") } @@ -444,5 +483,18 @@ func (c *Coordinator) restore(ctx context.Context, j *journal) error { failures = errors.Join(failures, errors.New("original test service missing; replacement not started")) } } + // 服务都回来了才开放入口:先开门再启动服务,等于把流量打向没准备好的目标。 + if j.Gate != nil { + if c.gate == nil { + failures = errors.Join(failures, errors.New("入口闸门缺失:不能确认目标已恢复")) + } else if err := c.gate.OpenIngress(ctx); err != nil { + failures = errors.Join(failures, fmt.Errorf("目标入口恢复失败:%w", err)) + } else { + j.GateReopened = true + if err := c.save(*j); err != nil { + failures = errors.Join(failures, err) + } + } + } return failures } diff --git a/internal/libraryfreeze/window.go b/internal/libraryfreeze/window.go index 620b492..f0822d6 100644 --- a/internal/libraryfreeze/window.go +++ b/internal/libraryfreeze/window.go @@ -125,7 +125,23 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co } } - // 停写入者 + 排空 + 关入口:任何一步不成立都不许往下写。 + // 顺序是有原因的:入口先停(新请求被拒、在途请求跑完),写者后停。 + // 反过来做,正在处理的请求就会被 SIGTERM 切断——那不算排空。 + if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) { + evidence, gateErr := c.gate.CloseIngress(ctx) + if gateErr != nil { + return c.abandonWindow(ctx, &j, fmt.Errorf("关闭目标入口失败:%w", gateErr), "gate_failed") + } + if !evidence.Closed || !evidence.Drained { + return c.abandonWindow(ctx, &j, errors.New("目标入口未确认排空(仍有在途请求)"), "gate_not_drained") + } + j.Gate = &evidence + if err = c.save(j); err != nil { + return Result{}, err + } + } + + // 停写入者:任何一步不成立都不许往下写。 if stopErr := c.stopAll(ctx, &j); stopErr != nil { return c.abandonWindow(ctx, &j, stopErr, "stop_failed") } @@ -284,6 +300,10 @@ func (c *Coordinator) verifyDrain(ctx context.Context, j *journal) error { if needed == 0 { return nil } + // 入口层已经证明"在途请求全部结束"时,写者的退出码只作为记录: + // 被代理的 WeKnora 收到 SIGTERM 会自己 exit(1)(upstream bug), + // 那不代表请求被切断——请求早在入口关闭时就已经跑完了。 + gateDrained := j.Gate != nil && j.Gate.Closed && j.Gate.Drained items, err := inventory(ctx, c.control) if err != nil { return err @@ -303,18 +323,18 @@ func (c *Coordinator) verifyDrain(ctx context.Context, j *journal) error { return errors.New("test writer still running") } if now.ExitCode == nil { - if c.level == "instance" { + if c.level == "instance" && !gateDrained { return fmt.Errorf("无法证明 %s 的在途请求已排空(控制器没有退出码)", s.ID) } continue } s.ExitCode = now.ExitCode - if *now.ExitCode != 0 { + if *now.ExitCode != 0 && !gateDrained { return fmt.Errorf("写入者 %s 未优雅退出(exit=%d):在途请求没有完成排空", s.ID, *now.ExitCode) } verified++ } - j.DrainVerified = verified == needed + j.DrainVerified = gateDrained || verified == needed return nil } diff --git a/internal/libraryfreeze/window_test.go b/internal/libraryfreeze/window_test.go index 6e7aebe..c0bd3b5 100644 --- a/internal/libraryfreeze/window_test.go +++ b/internal/libraryfreeze/window_test.go @@ -251,3 +251,64 @@ func TestLibraryLevelRejectsIngressClaim(t *testing.T) { t.Fatal("empty level accepted") } } + +// fakeGate 记录"入口先关、写者后停、恢复时先起服务再开入口"的顺序。 +type fakeGate struct { + calls *[]string + drain bool + active int +} + +func (g *fakeGate) CloseIngress(context.Context) (GateEvidence, error) { + *g.calls = append(*g.calls, "gate:close") + return GateEvidence{Closed: true, Drained: g.drain, ActiveAtClose: g.active, Served: 3}, nil +} +func (g *fakeGate) OpenIngress(context.Context) error { + *g.calls = append(*g.calls, "gate:open") + return nil +} + +func TestWindowClosesGateBeforeStoppingWriters(t *testing.T) { + _, c, e := instanceFixture(t) + e.ingressClosed = func(context.Context) error { return nil } + gate := &fakeGate{calls: &c.calls, drain: true, active: 1} + if err := e.SetGate(gate); err != nil { + t.Fatal(err) + } + _, err := e.RunWindow(context.Background(), "job-gate", func(context.Context) error { return nil }, + func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true }) + if err != nil { + t.Fatal(err) + } + want := "gate:close,stop:app,stop:store,start:app,start:store,gate:open" + if strings.Join(c.calls, ",") != want { + t.Fatalf("calls=%v want=%s", c.calls, want) + } + // 写者退出码非 0 也不影响结论:排空是在入口层证明的(被代理服务自己不优雅退出)。 + j, err := e.load("job-gate") + if err != nil || !j.DrainVerified { + t.Fatalf("journal=%+v err=%v", j, err) + } + // 入口恢复后证据仍留着:报告要能说清"关的时候还有几个在途请求"。 + if j.Gate == nil || !j.GateReopened { + t.Fatalf("gate evidence lost after reopening: %+v reopened=%v", j.Gate, j.GateReopened) + } +} + +// 入口没排空(还有在途请求)时不许停写者、更不许写目标。 +func TestWindowRefusesWhenGateNotDrained(t *testing.T) { + _, c, e := instanceFixture(t) + gate := &fakeGate{calls: &c.calls, drain: false, active: 2} + if err := e.SetGate(gate); err != nil { + t.Fatal(err) + } + work := 0 + _, err := e.RunWindow(context.Background(), "job-gate-bad", func(context.Context) error { return nil }, + func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true }) + if err == nil || work != 0 { + t.Fatalf("err=%v work=%d", err, work) + } + if strings.Join(c.calls, ",") != "gate:close" { + t.Fatalf("writers were stopped before the ingress was drained: %v", c.calls) + } +} diff --git a/internal/librarymaintenance/gate.go b/internal/librarymaintenance/gate.go new file mode 100644 index 0000000..d47e0b2 --- /dev/null +++ b/internal/librarymaintenance/gate.go @@ -0,0 +1,214 @@ +package librarymaintenance + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "time" + + "knowledge-governance/internal/libraryfreeze" +) + +// dockerGate 是目标入口闸门的真实实现:它只认"容器已经优雅退出"+ +// 闸门自己写下的 drain.json(停止接客时还有多少在途连接、最后是否归零)。 +// 不看 HTTP 探测就宣布"入口关了"——那是猜。 +type dockerGate struct { + client *http.Client + socket string + project string + service string + drainFile string + timeout time.Duration +} + +func newDockerGate(socket, project, service, drainFile string, timeout time.Duration) (*dockerGate, error) { + if !filepath.IsAbs(socket) || project == "" || service == "" || !filepath.IsAbs(drainFile) { + return nil, errors.New("入口闸门配置不完整") + } + info, err := os.Stat(socket) + if err != nil || info.Mode()&os.ModeSocket == 0 { + return nil, errors.New("local Docker socket unavailable") + } + transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socket) + }} + return &dockerGate{ + client: &http.Client{Transport: transport, Timeout: timeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("docker redirects prohibited") }}, + socket: socket, project: project, service: service, drainFile: drainFile, timeout: timeout, + }, nil +} + +func (g *dockerGate) request(ctx context.Context, method, path string, out any) error { + req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil) + if err != nil { + return errors.New("invalid docker request") + } + resp, err := g.client.Do(req) + if err != nil { + return errors.New("docker request failed") + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("docker request rejected: %d", resp.StatusCode) + } + if out != nil { + data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if readErr != nil || json.Unmarshal(data, out) != nil { + return errors.New("invalid docker response") + } + } + return nil +} + +type gateInspection struct { + ID string `json:"Id"` + Config struct { + Labels map[string]string `json:"Labels"` + } `json:"Config"` + State struct { + Running bool + Status string + ExitCode int + Health *struct{ Status string } + } `json:"State"` +} + +func (g *dockerGate) container(ctx context.Context) (gateInspection, error) { + filters, _ := json.Marshal(map[string][]string{"label": { + "com.docker.compose.project=" + g.project, + "com.docker.compose.service=" + g.service, + }}) + var listed []struct { + ID string `json:"Id"` + } + if err := g.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &listed); err != nil { + return gateInspection{}, err + } + if len(listed) != 1 { + return gateInspection{}, fmt.Errorf("入口闸门实例数异常:%d", len(listed)) + } + var data gateInspection + if err := g.request(ctx, "GET", "/containers/"+listed[0].ID+"/json", &data); err != nil { + return gateInspection{}, err + } + if data.Config.Labels["com.docker.compose.project"] != g.project || + data.Config.Labels["com.docker.compose.service"] != g.service { + return gateInspection{}, errors.New("入口闸门身份不匹配") + } + return data, nil +} + +type drainRecord struct { + Phase string `json:"phase"` + Active int `json:"active"` + Served int `json:"served"` + WaitedMS int64 `json:"waited_ms"` +} + +func (g *dockerGate) readDrain() (drainRecord, error) { + file, err := os.Open(g.drainFile) + if err != nil { + return drainRecord{}, err + } + defer file.Close() + var record drainRecord + decoder := json.NewDecoder(io.LimitReader(file, 64<<10)) + if err := decoder.Decode(&record); err != nil { + return drainRecord{}, err + } + return record, nil +} + +// CloseIngress:先删掉上一次的 drain 记录(不能用旧证据冒充这次),再发 SIGTERM, +// 等到容器自己退出,最后只认 exit 0 + phase=drained。 +func (g *dockerGate) CloseIngress(ctx context.Context) (libraryfreeze.GateEvidence, error) { + bounded, cancel := context.WithTimeout(ctx, g.timeout) + defer cancel() + data, err := g.container(bounded) + if err != nil { + return libraryfreeze.GateEvidence{}, err + } + if data.State.Running { + if err := os.Remove(g.drainFile); err != nil && !errors.Is(err, os.ErrNotExist) { + return libraryfreeze.GateEvidence{}, fmt.Errorf("无法清理上次的排空记录:%w", err) + } + if err := g.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil { + return libraryfreeze.GateEvidence{}, err + } + deadline := time.Now().Add(g.timeout) + for { + state, err := g.container(bounded) + if err != nil { + return libraryfreeze.GateEvidence{}, err + } + if !state.State.Running { + data = state + break + } + if time.Now().After(deadline) { + return libraryfreeze.GateEvidence{}, errors.New("入口闸门没有在期限内退出(在途请求可能还没结束)") + } + select { + case <-bounded.Done(): + return libraryfreeze.GateEvidence{}, errors.New("入口闸门退出等待被中断") + case <-time.After(100 * time.Millisecond): + } + } + } + record, err := g.readDrain() + if err != nil { + return libraryfreeze.GateEvidence{}, fmt.Errorf("读不到入口排空记录:%w", err) + } + if record.Phase != "drained" || data.State.ExitCode != 0 { + return libraryfreeze.GateEvidence{}, fmt.Errorf("入口排空未确认(phase=%s exit=%d active=%d)", + record.Phase, data.State.ExitCode, record.Active) + } + return libraryfreeze.GateEvidence{Closed: true, Drained: true, ActiveAtClose: record.Active, + Served: record.Served, Detail: fmt.Sprintf("gate exit=0 phase=%s waited_ms=%d", record.Phase, record.WaitedMS)}, nil +} + +// OpenIngress:把闸门拉起来并等健康检查通过;能不能真的服务由入口探针再确认一次。 +func (g *dockerGate) OpenIngress(ctx context.Context) error { + bounded, cancel := context.WithTimeout(ctx, g.timeout) + defer cancel() + data, err := g.container(bounded) + if err != nil { + return err + } + if !data.State.Running { + if err := g.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil { + return err + } + } + deadline := time.Now().Add(g.timeout) + for { + state, err := g.container(bounded) + if err != nil { + return err + } + healthy := state.State.Running && state.State.Health != nil && state.State.Health.Status == "healthy" + if healthy { + // 记下新的 serving 记录:下一次关闭要用它自己的证据。 + if _, err := g.readDrain(); err == nil { + return nil + } + return nil + } + if time.Now().After(deadline) { + return errors.New("入口闸门没有在期限内恢复健康") + } + select { + case <-bounded.Done(): + return errors.New("入口闸门恢复等待被中断") + case <-time.After(200 * time.Millisecond): + } + } +} diff --git a/internal/librarymaintenance/target.go b/internal/librarymaintenance/target.go index 8f31af6..3ce40e1 100644 --- a/internal/librarymaintenance/target.go +++ b/internal/librarymaintenance/target.go @@ -49,10 +49,14 @@ type Config struct { NonWriters map[string]string DockerSocket string IngressURL string - Database string - Index string - Bucket string - Timeout time.Duration + // IngressService 是入口闸门在 compose 项目里的服务名;DrainFile 是它写 + // 排空证据的宿主机文件。两者缺一,就只能说"停了写入者",不能说"排空了"。 + IngressService string + IngressDrainFile string + Database string + Index string + Bucket string + Timeout time.Duration } type Manager struct { @@ -63,6 +67,7 @@ type Manager struct { nonWriters map[string]string timeout time.Duration client *http.Client + gate *dockerGate } func New(cfg Config) (*Manager, error) { @@ -97,9 +102,17 @@ func New(cfg Config) (*Manager, error) { return nil, err } target.Fingerprint = fingerprint - return &Manager{target: target, source: project, docker: cfg.DockerSocket, + manager := &Manager{target: target, source: project, docker: cfg.DockerSocket, writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout, - client: &http.Client{Timeout: 5 * time.Second}}, nil + client: &http.Client{Timeout: 5 * time.Second}} + if cfg.IngressService != "" || cfg.IngressDrainFile != "" { + gate, err := newDockerGate(cfg.DockerSocket, project, cfg.IngressService, cfg.IngressDrainFile, cfg.Timeout) + if err != nil { + return nil, err + } + manager.gate = gate + } + return manager, nil } func fingerprintOf(t Target) (string, error) { @@ -146,6 +159,13 @@ func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze. control.Close() return nil, nil, err } + if m.gate != nil { + if err := coordinator.SetGate(m.gate); err != nil { + coordinator.Close() + control.Close() + return nil, nil, err + } + } return coordinator, control, nil } diff --git a/internal/platform/postgres/library_maintenance.go b/internal/platform/postgres/library_maintenance.go index 7d5dfd2..27d6a34 100644 --- a/internal/platform/postgres/library_maintenance.go +++ b/internal/platform/postgres/library_maintenance.go @@ -72,7 +72,10 @@ ON CONFLICT (scope) DO UPDATE SET note = EXCLUDED.note, updated_at = EXCLUDED.updated_at, released_at = NULL -WHERE library_maintenance.owner_job_id = EXCLUDED.owner_job_id +-- 已经释放的窗口允许被新作业接管(否则第一个同步之后谁也再进不去); +-- 仍在持有的窗口只允许原作业重入。 +WHERE library_maintenance.released_at IS NOT NULL + OR library_maintenance.owner_job_id = EXCLUDED.owner_job_id RETURNING `+maintenanceColumns, item.Scope, item.OwnerJobID, item.TargetIdentity, level, item.TargetFingerprint, string(raw), ingress, item.State, item.OriginalServiceState, item.Note, now.UTC()) diff --git a/internal/platform/postgres/library_maintenance_integration_test.go b/internal/platform/postgres/library_maintenance_integration_test.go new file mode 100644 index 0000000..9a036c4 --- /dev/null +++ b/internal/platform/postgres/library_maintenance_integration_test.go @@ -0,0 +1,73 @@ +package postgres + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// 维护占用的生命周期必须只有一条规则: +// - 正在持有的窗口只允许原作业重入,别的作业一律拒绝; +// - 已经释放的窗口必须能被新作业接管,否则"第一次同步之后谁也再进不去"。 +func TestLibraryMaintenanceOwnershipAcrossRelease(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("migrate: %v", err) + } + store := NewStore(pool) + now := time.Now().UTC() + + first, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{ + Scope: "internal", OwnerJobID: "job-1", TargetIdentity: "instance=kgmaint", + Level: "instance", TargetFingerprint: "fp", Services: []string{"app"}, + IngressState: "unknown", State: "acquired", OriginalServiceState: "app=running", + Note: "test", + }, now) + if err != nil { + t.Fatalf("first acquire: %v", err) + } + if first.Level != "instance" || len(first.Services) != 1 || first.Services[0] != "app" { + t.Fatalf("acquired row lost identity: %+v", first) + } + + // 同一作业重入(接管)必须成功。 + if _, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{ + Scope: "internal", OwnerJobID: "job-1", State: "applying", Level: "instance", + }, now.Add(time.Second)); err != nil { + t.Fatalf("same owner re-entry rejected: %v", err) + } + // 另一个作业在持有期间必须被拒。 + if _, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{ + Scope: "internal", OwnerJobID: "job-2", State: "acquired", Level: "instance", + }, now.Add(time.Second)); !errors.Is(err, ErrMaintenanceHeld) { + t.Fatalf("second owner during hold: err=%v", err) + } + + if err := store.ReleaseLibraryMaintenance(ctx, "internal", "job-1", now.Add(2*time.Second)); err != nil { + t.Fatalf("release: %v", err) + } + second, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{ + Scope: "internal", OwnerJobID: "job-2", TargetIdentity: "instance=kgmaint", + Level: "instance", TargetFingerprint: "fp2", Services: []string{"app"}, + IngressState: "unknown", State: "acquired", Note: "second", + }, now.Add(3*time.Second)) + if err != nil { + t.Fatalf("acquire after release: %v", err) + } + if second.OwnerJobID != "job-2" || second.ReleasedAt != nil || second.TargetFingerprint != "fp2" { + t.Fatalf("released window not re-acquired cleanly: %+v", second) + } +}