From 3643e9bf82f01ce1b78ecc0a52b67a6ef2f77d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Thu, 29 Oct 2026 00:50:32 +0800 Subject: [PATCH] =?UTF-8?q?fix(library):=20=E7=BB=B4=E6=8A=A4=E6=94=B6?= =?UTF-8?q?=E5=B0=BE=E5=BF=85=E9=A1=BB=E6=8C=89=E9=A2=84=E5=A4=87=E4=BB=BD?= =?UTF-8?q?=E6=81=A2=E5=A4=8D=E5=B9=B6=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。 新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下 执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。 预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者; 写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。 --- CHANGELOG.md | 34 +++ api/openapi.yaml | 52 ++++ cmd/api/main.go | 1 + cmd/libraryctl/main.go | 31 +-- cmd/worker/library_handlers.go | 73 ++++-- cmd/worker/library_recovery.go | 163 ++++++++++++ cmd/worker/main.go | 9 + internal/appconfig/config.go | 12 +- internal/httpapi/library_flow.go | 103 ++++++++ internal/httpapi/library_restore.go | 4 + internal/httpapi/server.go | 20 +- internal/libraryfreeze/freeze.go | 79 +++++- internal/libraryfreeze/recover_window_test.go | 175 ++++++++++++ internal/libraryfreeze/recovery.go | 141 ++++++++++ internal/libraryfreeze/window.go | 123 ++++----- internal/libraryfreeze/window_test.go | 42 +-- internal/librarymaintenance/dockerapi.go | 141 ++++++++++ internal/librarymaintenance/gate.go | 93 ++----- internal/librarymaintenance/quiesce.go | 248 ++++++++++++++++++ internal/librarymaintenance/target.go | 128 ++++++++- internal/librarymaintenance/target_test.go | 21 ++ .../postgres/library_instance_maintenance.go | 177 +++++++++++++ .../library_maintenance_integration_test.go | 42 +++ ...0072_library_instance_maintenance.down.sql | 1 + ...000072_library_instance_maintenance.up.sql | 21 ++ internal/platform/postgres/migrations_test.go | 21 +- 26 files changed, 1730 insertions(+), 225 deletions(-) create mode 100644 cmd/worker/library_recovery.go create mode 100644 internal/libraryfreeze/recover_window_test.go create mode 100644 internal/libraryfreeze/recovery.go create mode 100644 internal/librarymaintenance/dockerapi.go create mode 100644 internal/librarymaintenance/quiesce.go create mode 100644 internal/platform/postgres/library_instance_maintenance.go create mode 100644 internal/platform/postgres/migrations/000072_library_instance_maintenance.down.sql create mode 100644 internal/platform/postgres/migrations/000072_library_instance_maintenance.up.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index ae8dcbb..8522e83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## 开发快照(维护收尾必须按预备份验证 / 后台任务静默 / 实例级互斥) + +### Fixed + +- **删除 `FinalizeAfterOperatorRecovery`**:它自称"不启动服务",实际会 `gate.OpenIngress`, + 而且只看服务健康与入口 200——健康不等于恢复成功,这条路径能把半写内容重新开放。 +- 新的唯一收尾入口 `Coordinator.RecoverWindow`:绑定「原作业 + 目标指纹 + journal 里持久化的预备份身份」, + 在**维护关闭**状态下(入口关、写入者停、排空确认)执行 `Restore` 与**逐类校验** + (行数/身份集合/逐行内容摘要 + 索引文档 + 对象资源),校验通过才恢复服务、开入口; + 校验或恢复失败则保持维护(服务停、入口关、journal 非终态)。 +- 预备份身份与「内容风险」在**第一笔写入之前一起落盘**(`Window.SetBackupSnapshot` + + `MarkContentRisk`)。没有预备份身份的窗口之后既不能自动恢复,也不许被任何收尾路径开放。 +- 低层恢复接口不再接受任何"已恢复/已验证"的入参:`Restorer`/`Verifier` 由协调器在窗口内调用, + `Verifier` 必须返回校验摘要并被持久化。 + +### Added + +- 真实恢复入口 `POST /api/v1/library/maintenance/recover`:只建 `library.recover` 作业, + 由 worker 按预备份恢复 + 校验后释放占用;`libraryctl finalize` 删除(不再有"手工收尾"旁路)。 +- 实例级互斥:迁移 `000072_library_instance_maintenance`,维护占用按**实例**建键, + internal 持有期间 external 也进不来(旧实现按 scope 建行,两个 scope 会同时停/起同一实例)。 +- 后台静默检查(`targetQuiescer`):窗口关闭入口之前,必须证明目标库 `task_pending_ops=0`、 + 无未完成解析、asynq 队列(pending/active/scheduled/retry)为空;否则拒绝开窗口。 + HTTP 排空不等于后台任务排空——app 内的 asynq worker 会被上游 SIGTERM 的 exit(1) 切断。 +- 唯一入口拓扑校验(`Manager.VerifyTopology`):写入者**不许发布任何宿主机端口**, + 入口必须是独立声明的闸门且发布的端口与入口 URL 一致;不满足就拒绝开窗口。 + +### 限制(本轮不承诺) + +- 拓扑校验只看"声明的写入者组":新增容器必须显式声明,平台不会自动猜测。 +- 后台静默检查依赖目标库表 `task_pending_ops/knowledges` 与 asynq 键名(v0.7.1 现状); + 上游改队列实现时需要同步调整。 +- 页面只透传新增字段(实例占用/入口状态),专门文案与浏览器验收仍未做。 + ## 开发快照(目标维护:实例级窗口) ### Added diff --git a/api/openapi.yaml b/api/openapi.yaml index 3ba2fc8..50f3501 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1062,6 +1062,58 @@ } } }, + "/api/v1/library/maintenance/recover": { + "post": { + "tags": [ + "Runtime" + ], + "operationId": "recoverLibraryMaintenance", + "summary": "按预备份恢复并逐类校验后收尾维护窗口(仅管理员;只建恢复作业,恢复在校验通过前不开放目标)", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "type": "string", + "enum": [ + "internal", + "external" + ] + }, + "window_job_id": { + "type": "string", + "description": "可选:必须与当前维护窗口的作业一致" + } + } + } + } + } + }, + "responses": { + "202": { + "description": "已创建恢复作业并入队" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "description": "没有未释放的维护窗口,或窗口属于别的 scope/作业" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "description": "目标维护未接线或维护记录存储不可用" + } + } + } + }, "/api/v1/library/restores": { "post": { "tags": [ diff --git a/cmd/api/main.go b/cmd/api/main.go index 296c4fa..4b8e8c7 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -202,6 +202,7 @@ func run(ctx context.Context, getenv func(string) string) error { VersionHistory: versionHistoryStore, VersionHistoryRepo: versionHistoryRepo, LibraryArchiveRoot: config.LibraryArchiveRoot, + LibraryTargetInstance: config.LibraryTargetInstance, LibraryAgentSocket: config.LibraryAgentSocket, LibraryCapacityBudgetBytes: config.LibraryCapacityBudgetBytes, LibraryProductionDatabaseURL: config.LibraryProductionDatabaseURL, diff --git a/cmd/libraryctl/main.go b/cmd/libraryctl/main.go index 92da4e3..b8a82f2 100644 --- a/cmd/libraryctl/main.go +++ b/cmd/libraryctl/main.go @@ -90,8 +90,6 @@ func main() { runErr = runVerify(opts) case "recover": runErr = runRecover(ctx, opts) - case "finalize": - runErr = runFinalize(ctx, opts) default: usage() os.Exit(2) @@ -103,7 +101,7 @@ func main() { } func usage() { - fmt.Fprint(os.Stderr, `libraryctl [flags] + fmt.Fprint(os.Stderr, `libraryctl [flags] status probe the real source and print what is configured / blocked preflight read the source, run settlement + candidate validation (no writes, no stop) @@ -111,8 +109,6 @@ func usage() { list list sealed candidates and stored bytes verify re-verify one sealed candidate (digests, coverage, identity) recover restore writers from an interrupted freeze journal - finalize close a held window after an operator recovered the target by hand - (verifies services + ingress are really serving; starts nothing) Flags (defaults target the local WKNFull test stack): -source-app WKNFull-app container whose env describes the source @@ -821,31 +817,6 @@ func runVerify(opts options) error { return nil } -func runFinalize(ctx context.Context, opts options) error { - if opts.job == "" { - return errors.New("finalize requires -job") - } - coordinator, control, err := openFreeze(ctx, opts) - if err != nil { - return err - } - defer coordinator.Close() - defer control.Close() - if err := coordinator.FinalizeAfterOperatorRecovery(ctx, opts.job); err != nil { - return err - } - items, err := control.Inventory(ctx) - if err != nil { - return err - } - group := []map[string]any{} - for _, item := range items { - group = append(group, map[string]any{"id": item.ID, "running": item.Running, "healthy": item.Healthy}) - } - printJSON(map[string]any{"finalized": opts.job, "code": "operator_recovered", "writers": group}) - return nil -} - func runRecover(ctx context.Context, opts options) error { coordinator, control, err := openFreeze(ctx, opts) if err != nil { diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index 154e2cf..5dec9c9 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -182,6 +182,12 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup") } + defer manager.Close() + // 拓扑必须先过关:写入者发布了端口 = 存在绕闸直连,窗口关的只是半扇门。 + if topoErr := manager.VerifyTopology(ctx); topoErr != nil { + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, + "目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup") + } targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope, redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket) originalState, stateErr := manager.OriginalState(ctx) @@ -190,6 +196,19 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a "读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup") } identity := manager.Target() + // 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。 + instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{ + Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope, + TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance, + Services: identity.Writers, IngressState: "unknown", State: "acquired", + OriginalServiceState: originalState, + Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放", + }, now) + if instanceErr != nil { + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, + "目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "prebackup") + } + _ = instanceLock maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{ Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity + " | " + manager.Description(), @@ -202,6 +221,8 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放", }, now) if acquireErr != nil { + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", + "scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now) return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup") } _ = maintenance @@ -213,10 +234,11 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a if resumed { _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", "维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC()) + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", + "维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC()) } else { - _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified", + _ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, "未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC()) - _ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC()) } return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "维护协调器不可用:"+openErr.Error(), "prebackup") } @@ -348,8 +370,13 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a return manager.VerifyRunning(ctx) } windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) { - if err := w.MarkContentRisk(ctx); err != nil { - return "", err + // 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘:没有预备份身份的 + // 窗口之后既不能自动恢复,也不许被任何收尾路径开放。 + recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error { + if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil { + return err + } + return w.MarkContentRisk(backupCtx) } if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed", "目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil { @@ -369,39 +396,46 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a note += ":" + windowErr.Error() } _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2) + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2) if windowErr == nil { return errors.New(note) } return windowErr case contentState == contentSucceeded: - _ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "目标入口已恢复并验证", now2) - _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified", - "写入与校验完成,目标已按原运行状态恢复", now2) - if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil { + if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, + "写入与校验完成,目标已按原运行状态恢复", now2); err != nil { return fmt.Errorf("目标已恢复但维护占用释放失败: %w", err) } return windowErr case contentState == contentRolledBack: - _ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "已回滚并验证,目标入口恢复", now2) - _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified", - "本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2) - if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil { + if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, + "本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2); err != nil { return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err) } return windowErr default: // 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。 - _ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", - "未进入维护窗口,目标未被停用", now2) - _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified", - "未写目标,维护窗口未开启", now2) - if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil { + if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, + "未写目标,维护窗口未开启", now2); err != nil { return fmt.Errorf("维护占用释放失败: %w", err) } return windowErr } } +// releaseWindowLocks 是唯一释放路径:先记录入口已恢复,再放 scope 行与实例锁。 +// 顺序反过来会出现"实例锁放了、scope 还挂着"的中间态,页面就会说正在维护。 +func releaseWindowLocks(ctx context.Context, store *postgresstore.Store, instance, scope, jobID, note string, now time.Time) error { + _ = store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now) + _ = store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now) + _ = store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now) + _ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now) + if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); err != nil { + return err + } + return store.ReleaseLibraryMaintenance(ctx, scope, jobID, now) +} + // resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业: // 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用), // 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。 @@ -485,6 +519,11 @@ func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.Curren // 上游据此在写目标前中止,保证"没有可恢复基线就不写"。 func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error { now := time.Now().UTC() + if r.onBackup != nil { + if err := r.onBackup(ctx, backup); err != nil { + return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err) + } + } if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "", backup.SnapshotID, "", now); err != nil { return fmt.Errorf("持久化预备份引用失败: %w", err) diff --git a/cmd/worker/library_recovery.go b/cmd/worker/library_recovery.go new file mode 100644 index 0000000..439d641 --- /dev/null +++ b/cmd/worker/library_recovery.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log" + "time" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + "knowledge-governance/internal/libraryrelease" + postgresstore "knowledge-governance/internal/platform/postgres" + v071 "knowledge-governance/internal/releaseadapter/v071" +) + +// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。 +// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。 +type libraryRecoveryPayload struct { + Scope string `json:"scope"` + WindowJobID string `json:"window_job_id"` +} + +// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口: +// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源), +// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。 +func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error { + var payload libraryRecoveryPayload + if err := json.Unmarshal(job.Payload, &payload); err != nil || + (payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" { + return errors.New("invalid library recovery payload") + } + manager, err := openLibraryMaintenance(config) + if err != nil { + return fmt.Errorf("目标维护未接线,不能做恢复:%w", err) + } + defer manager.Close() + identity := manager.Target() + if topoErr := manager.VerifyTopology(ctx); topoErr != nil { + return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr) + } + + lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance) + if err != nil { + return fmt.Errorf("读不到实例维护记录:%w", err) + } + if lock == nil || lock.ReleasedAt != nil { + return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复") + } + if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope { + return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope) + } + if lock.TargetFingerprint != identity.Fingerprint { + return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标") + } + + // 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。 + windowJob, err := store.LibraryJob(ctx, payload.WindowJobID) + if err != nil || windowJob.SnapshotID == "" { + return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份") + } + prebackup := windowJob.SnapshotID + knownMapping := knownIndexMapping(ctx, store, payload.Scope) + target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping) + if wireErr != nil { + return fmt.Errorf("恢复目标装配失败:%w", wireErr) + } + defer target.Close() + + coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot) + if openErr != nil { + return fmt.Errorf("维护协调器不可用:%w", openErr) + } + defer control.Close() + defer coordinator.Close() + + restorer := recoveryRestorer{target: target, expected: prebackup} + verifier := recoveryVerifier{target: target, expected: prebackup} + result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier) + now := time.Now().UTC() + if recoverErr != nil { + note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error() + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now) + return recoverErr + } + if result.State != "recovered" { + note := "恢复流程结束但状态不是 recovered:" + result.State + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) + return errors.New(note) + } + if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID, + "按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil { + return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err) + } + log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup) + return nil +} + +// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致 +// 就拒绝——那是两份快照,恢复出来的东西不能算数。 +type recoveryRestorer struct { + target *v071.ProductionLibraryTarget + expected string +} + +func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error { + if backupID != r.expected { + return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected) + } + return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID}) +} + +// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。 +type recoveryVerifier struct { + target *v071.ProductionLibraryTarget + expected string +} + +func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) { + if backupID != v.expected { + return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected) + } + // Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。 + if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil { + return "", err + } + state, err := v.target.Current(ctx) + if err != nil { + return "", fmt.Errorf("恢复后读不到目标状态:%w", err) + } + raw, err := json.Marshal(state) + if err != nil { + return "", err + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]), nil +} + +// knownIndexMapping 取"上一次登记过的索引映射",作为装配目标时的映射基线。 +func knownIndexMapping(ctx context.Context, store *postgresstore.Store, scope string) string { + versions, err := store.LibraryVersions(ctx, scope, 50) + if err != nil { + return "" + } + for _, item := range versions { + if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil { + if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" { + return raw + } + } + } + for _, item := range versions { + if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" { + return raw + } + } + return "" +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 5dacc34..814e7a8 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -18,6 +18,7 @@ import ( "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryagent" + "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarysnapshot" postgresstore "knowledge-governance/internal/platform/postgres" "knowledge-governance/internal/platform/weknora" @@ -36,6 +37,8 @@ type libraryStageRecorder struct { totals map[string]stageSize // mappingDigest 是本次读到的生产索引映射,随预备份一起登记成版本证据。 mappingDigest string + // onBackup 在预备份完成后、写目标之前调用:把"预备份身份 + 内容风险"落盘。 + onBackup func(context.Context, libraryrelease.BackupRef) error } func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail string) { @@ -100,6 +103,9 @@ const libraryCaptureJobType = "library.capture" // librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。 const librarySyncJobType = "library.sync" +// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型;与 API 侧常量同值。 +const libraryRecoverJobType = "library.recover" + // libraryRestoreJobType 是按指定整库版本恢复的队列类型;与同步分开、授权也分开。 const libraryRestoreJobType = "library.restore" @@ -177,6 +183,9 @@ func run(ctx context.Context, getenv func(string) string) error { handlers[libraryRestoreJobType] = func(ctx context.Context, job jobs.Job) error { return runLibraryRelease(ctx, store, config, job, governance.LibraryJobRestore) } + handlers[libraryRecoverJobType] = func(ctx context.Context, job jobs.Job) error { + return runLibraryRecovery(ctx, store, config, job) + } worker, err := jobs.NewWorker(jobs.WorkerConfig{ Queue: store, diff --git a/internal/appconfig/config.go b/internal/appconfig/config.go index f11f362..097703d 100644 --- a/internal/appconfig/config.go +++ b/internal/appconfig/config.go @@ -105,7 +105,11 @@ type Config struct { // 入口闸门:关入口与排空在它身上做(被代理服务自己不排空)。 LibraryTargetIngressService string LibraryTargetIngressDrainFile string - LibraryTargetJournalRoot string + // 后台静默检查:目标队列 + 目标库(HTTP 排空不等于后台任务排空)。 + LibraryTargetRedisAddr string + LibraryTargetRedisPassword string + LibraryTargetRedisDB int + LibraryTargetJournalRoot string // LibraryAgentSocket 是主机侧封存 agent 的 unix socket(只由 worker 连接)。 // 留空即"未接线":页面不显示可用的封存入口,接口返回原因而不是假装已排队。 LibraryAgentSocket string @@ -202,6 +206,9 @@ func Load(getenv func(string) string) (Config, error) { LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")), LibraryTargetIngressService: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_SERVICE")), LibraryTargetIngressDrainFile: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_DRAIN_FILE")), + LibraryTargetRedisAddr: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_ADDR")), + LibraryTargetRedisPassword: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_PASSWORD")), + LibraryTargetRedisDB: valueOrDefaultInt(getenv("LIBRARY_TARGET_REDIS_DB"), 0), LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")), LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"), LibraryEmbeddingDimension: embeddingDimension, @@ -349,6 +356,9 @@ func Load(getenv func(string) string) (Config, error) { if config.LibraryTargetIngressDrainFile == "" { missing = append(missing, "LIBRARY_TARGET_INGRESS_DRAIN_FILE") } + if config.LibraryTargetRedisAddr == "" { + missing = append(missing, "LIBRARY_TARGET_REDIS_ADDR") + } if len(missing) > 0 { return Config{}, fmt.Errorf("目标维护配置不完整:缺少 %s", strings.Join(missing, "、")) } diff --git a/internal/httpapi/library_flow.go b/internal/httpapi/library_flow.go index c3175b4..b3e1d16 100644 --- a/internal/httpapi/library_flow.go +++ b/internal/httpapi/library_flow.go @@ -42,6 +42,9 @@ type libraryVersionView struct { // librarySyncJobType 与 worker 的处理器类型一致;单独定义避免 API 依赖 worker 包。 const librarySyncJobType = "library.sync" +// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型(同一份定义也在 worker 里)。 +const libraryRecoverJobType = "library.recover" + type libraryKnownView struct { State string `json:"state"` // known | unknown Version *libraryVersionView `json:"version,omitempty"` @@ -291,6 +294,101 @@ func libraryStore(config Config) (libraryJobStore, bool) { return store, ok } +// instanceMaintenanceGuard 在"实例级维护"被别的作业持有时拒绝新的写入请求。 +// 只看 scope 行是不够的:internal 与 external 动的是同一批容器。 +func instanceMaintenanceGuard(ctx context.Context, config Config, scope string) (map[string]any, bool) { + instance := config.LibraryTargetInstance + if instance == "" { + return nil, false + } + store, ok := config.Store.(interface { + InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error) + }) + if !ok { + return nil, false + } + held, err := store.InstanceMaintenanceOf(ctx, instance) + if err != nil || held == nil || held.ReleasedAt != nil { + return nil, false + } + return map[string]any{"code": "library_target_in_maintenance", + "error": "目标实例正在维护中,验证完成前不接受新的写入请求", + "instance": held.Instance, "owner_job": held.OwnerJobID, "owner_scope": held.OwnerScope, + "state": held.State, "ingress_state": held.IngressState}, true +} + +// libraryMaintenanceRecoverHandler 是"按预备份恢复并校验"的唯一入口: +// 它只建一个恢复作业,真正的恢复与校验由 worker 在维护关闭状态下执行。 +func libraryMaintenanceRecoverHandler(w http.ResponseWriter, r *http.Request, config Config) { + if r.Method != http.MethodPost { + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"}) + return + } + var request struct { + Scope string `json:"scope"` + WindowJobID string `json:"window_job_id"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + if request.Scope != "internal" && request.Scope != "external" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"}) + return + } + instance := config.LibraryTargetInstance + if instance == "" { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_recovery_unavailable", + "error": "目标维护未接线:无法执行按预备份恢复"}) + return + } + store, ok := config.Store.(interface { + InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error) + }) + if !ok { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "维护记录存储不可用"}) + return + } + held, err := store.InstanceMaintenanceOf(r.Context(), instance) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护记录"}) + return + } + if held == nil || held.ReleasedAt != nil { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_no_held_window", + "error": "目标实例当前没有未释放的维护窗口,不需要恢复"}) + return + } + if held.OwnerScope != request.Scope { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_scope_mismatch", + "error": "维护窗口属于另一个 scope", "owner_scope": held.OwnerScope}) + return + } + if request.WindowJobID != "" && request.WindowJobID != held.OwnerJobID { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_mismatch", + "error": "维护窗口属于另一个作业", "owner_job": held.OwnerJobID}) + return + } + now := time.Now().UTC() + payload, err := json.Marshal(map[string]string{"scope": held.OwnerScope, "window_job_id": held.OwnerJobID}) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build recovery payload"}) + return + } + // 队列作业 id 与幂等键都按窗口固定:重复点击只会得到同一个恢复作业。 + queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{ + ID: "librecover-" + held.OwnerJobID, Type: libraryRecoverJobType, Payload: payload, + AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.recover:" + held.OwnerJobID, + }, now) + if err != nil || queueJob == nil { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_recovery_enqueue_failed", + "error": "恢复作业投递失败(或已有同窗口的恢复作业)", "detail": errString(err)}) + return + } + writeJSON(w, http.StatusAccepted, map[string]any{"job": queueJob, "window_job_id": held.OwnerJobID, + "scope": held.OwnerScope, "instance": held.Instance}) +} + // activeGrantFor 只查"目标+快照+同一参数摘要"的未过期未撤销授权。 func activeGrantFor(ctx context.Context, config Config, scope, snapshotID string) (governance.LibraryReleaseAuthorization, bool) { if snapshotID == "" { @@ -597,6 +695,11 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { } grantFound, hasGrant := activeGrantFor(r.Context(), config, request.Scope, request.Target) _ = grantFound + // 目标实例正在维护(跨对内/对外互斥):直接拒绝,不等到 worker 里才发现。 + if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held { + writeJSON(w, http.StatusConflict, body) + return + } // 目标正在维护(被别的作业持有):直接拒绝新写入,不等到 worker 里才发现。 if maintenanceStore, ok := config.Store.(interface { LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error) diff --git a/internal/httpapi/library_restore.go b/internal/httpapi/library_restore.go index 311f142..10f28f3 100644 --- a/internal/httpapi/library_restore.go +++ b/internal/httpapi/library_restore.go @@ -54,6 +54,10 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"}) return } + if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held { + writeJSON(w, http.StatusConflict, body) + return + } digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID) if err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"}) diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 1784dad..5f7fe6c 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -145,6 +145,9 @@ type Config struct { VersionHistoryRepo string // LibraryArchiveRoot 是整库封存产物的只读归档根;留空即未接线(页面降级)。 LibraryArchiveRoot string + // LibraryTargetInstance 是实例级维护的目标实例名;留空即"未接线", + // 页面不能发起按预备份恢复,接口返回原因而不是假装能恢复。 + LibraryTargetInstance string // LibraryAgentSocket 是主机侧封存 agent 的 unix socket;留空即页面不能发起封存。 LibraryAgentSocket string // LibraryCapacityBudgetBytes 是整库归档容量预算;0 = 未设置(容量门禁不放行)。 @@ -461,7 +464,22 @@ func NewHandler(config Config) (http.Handler, error) { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护窗口"}) return } - writeJSON(w, http.StatusOK, map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held}) + response := map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held} + // 实例级占用才是真正的互斥来源:跨 scope 也在这里体现。 + if instanceStore, ok := config.Store.(interface { + InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error) + }); ok && config.LibraryTargetInstance != "" { + if instanceHeld, err := instanceStore.InstanceMaintenanceOf(r.Context(), config.LibraryTargetInstance); err == nil { + response["instance"] = config.LibraryTargetInstance + response["instance_maintenance"] = instanceHeld + response["held"] = instanceHeld != nil && instanceHeld.ReleasedAt == nil + } + } + writeJSON(w, http.StatusOK, response) + }) + // 按预备份恢复并逐类校验:维护窗口卡住时唯一的收尾入口(worker 执行)。 + mux.HandleFunc("POST /api/v1/library/maintenance/recover", func(w http.ResponseWriter, r *http.Request) { + libraryMaintenanceRecoverHandler(w, r, config) }) mux.HandleFunc("GET /api/v1/library/jobs", func(w http.ResponseWriter, r *http.Request) { libraryJobsHandler(w, r, config) diff --git a/internal/libraryfreeze/freeze.go b/internal/libraryfreeze/freeze.go index 1b3e232..cc5643c 100644 --- a/internal/libraryfreeze/freeze.go +++ b/internal/libraryfreeze/freeze.go @@ -47,6 +47,15 @@ type Gate interface { OpenIngress(context.Context) error } +// Quiescer 回答"后台写入者是不是已经空了"。 +// +// 为什么 HTTP 排空不够:被代理的 WeKnora 在 SIGTERM 后会 exit(1),app 内的 asynq +// worker 正在跑的后台任务会被直接切断。要么等它们跑完,要么明确不进入窗口。 +type Quiescer interface { + // Idle 返回 true 表示此刻没有排队/在跑的后台任务,也没有未结清的处理记录。 + Idle(ctx context.Context) (bool, string, error) +} + // Inventory must enumerate the complete, independently scoped test writer set, // using immutable instance IDs. Never implement it as only the currently running // subset. Start must wait for readiness; callers still read back the result. @@ -81,12 +90,23 @@ type journal struct { // Gate 记录入口关闭/排空的证据;没有它就不能说"入口关了"。 // 入口恢复后证据仍保留(GateReopened=true),否则事后查不到 // "关的时候还有几个在途请求"。 - Gate *GateEvidence `json:"gate,omitempty"` - GateReopened bool `json:"gate_reopened,omitempty"` - Services []serviceRecord `json:"services"` - StartedAt time.Time `json:"started_at"` - FinishedAt *time.Time `json:"finished_at,omitempty"` - Code string `json:"code,omitempty"` + Gate *GateEvidence `json:"gate,omitempty"` + GateReopened bool `json:"gate_reopened,omitempty"` + // BackupSnapshotID 是这次窗口开始写之前落盘的那份预备份:恢复只能按它来, + // 不允许事后换一份"看起来更好"的快照。 + BackupSnapshotID string `json:"backup_snapshot_id,omitempty"` + // TargetFingerprint 是开窗口时的目标身份:恢复时必须还是同一个目标。 + TargetFingerprint string `json:"target_fingerprint,omitempty"` + // BackgroundIdle 记录"停写入者之前后台任务队列确认为空"。 + BackgroundIdle bool `json:"background_idle,omitempty"` + // 恢复证据:只有真正跑完"恢复 + 校验"才会写这三项。 + RecoveredBackup string `json:"recovered_backup,omitempty"` + RecoveryDigest string `json:"recovery_digest,omitempty"` + RecoveredAt *time.Time `json:"recovered_at,omitempty"` + Services []serviceRecord `json:"services"` + StartedAt time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Code string `json:"code,omitempty"` } type Coordinator struct { root *os.Root @@ -99,6 +119,9 @@ type Coordinator struct { ingressClosed func(context.Context) error ingressReady func(context.Context) error gate Gate + quiescer Quiescer + fingerprint string + quiesceWait time.Duration } func Open(root, source string, c Controller) (*Coordinator, error) { @@ -149,6 +172,37 @@ func (c *Coordinator) SetGate(g Gate) error { return nil } +// SetTargetFingerprint 绑住"这次维护动的是哪个目标"。恢复时必须与 journal +// 里记录的一致,否则说明配置换了目标,禁止拿旧窗口去收尾新目标。 +func (c *Coordinator) SetTargetFingerprint(fingerprint string) error { + if len(fingerprint) != 64 { + return errors.New("invalid target fingerprint") + } + if _, err := hex.DecodeString(fingerprint); err != nil { + return errors.New("invalid target fingerprint") + } + c.fingerprint = strings.ToLower(fingerprint) + return nil +} + +// SetQuiesceTimeout 调整"等后台任务静默"的上限(默认 2 分钟)。 +func (c *Coordinator) SetQuiesceTimeout(d time.Duration) error { + if d <= 0 { + return errors.New("invalid quiesce timeout") + } + c.quiesceWait = d + return nil +} + +// SetQuiescer 声明后台任务静默检查(HTTP 排空不等于后台任务排空)。 +func (c *Coordinator) SetQuiescer(q Quiescer) error { + if q == nil { + return errors.New("invalid quiescer") + } + c.quiescer = q + return nil +} + func validIngress(raw string) bool { parsed, err := url.Parse(raw) return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.Path != "" && parsed.User == nil @@ -252,7 +306,7 @@ func validID(s string) bool { } return true } -func terminal(s string) bool { return s == "succeeded" || s == "failed" } +func terminal(s string) bool { return s == "succeeded" || s == "failed" || s == "recovered" } // A kernel lock serializes the whole test writer group, not individual KBs. // A crash releases the lock, but leaves the write-ahead journal for recovery. @@ -328,7 +382,7 @@ func (c *Coordinator) load(job string) (journal, error) { return journal{}, errors.New("freeze journal identity mismatch") } switch j.State { - case "stopping", "held", "capturing", "restoring", "succeeded", "failed": + case "stopping", "held", "capturing", "recovering", "restoring", "succeeded", "failed", "recovered": default: return journal{}, errors.New("invalid freeze journal state") } @@ -351,6 +405,15 @@ func (c *Coordinator) load(job string) (journal, error) { if j.GateReopened && j.Gate == nil { return journal{}, errors.New("ingress reopened without close evidence") } + if j.BackupSnapshotID != "" && !validDigest(j.BackupSnapshotID) { + return journal{}, errors.New("invalid backup snapshot id") + } + // 恢复证据只允许指向"这次窗口自己那份预备份",且必须带校验摘要。 + if j.RecoveredAt != nil { + if j.RecoveredBackup == "" || j.RecoveredBackup != j.BackupSnapshotID || j.RecoveryDigest == "" { + return journal{}, errors.New("invalid recovery evidence") + } + } if len(j.Services) == 0 || len(j.Services) > 256 { return journal{}, errors.New("invalid freeze journal services") } diff --git a/internal/libraryfreeze/recover_window_test.go b/internal/libraryfreeze/recover_window_test.go new file mode 100644 index 0000000..b2fc1bf --- /dev/null +++ b/internal/libraryfreeze/recover_window_test.go @@ -0,0 +1,175 @@ +package libraryfreeze + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +func heldJournal(job string, fingerprint, backup string) journal { + j := journal{Version: 1, Source: "target-instance", Job: job, Level: "instance", + Ingress: "http://127.0.0.1:1/health", TargetFingerprint: fingerprint, + ContentRisk: true, RestoreBlocked: true, BackupSnapshotID: backup, + Result: Result{State: "held"}, StartedAt: time.Now().UTC(), + Gate: &GateEvidence{Closed: true, Drained: true}} + j.Services = []serviceRecord{ + {Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true, StopVerified: true}, + {Service: Service{ID: "store", Running: true, Healthy: true}, StopRequested: true, StopVerified: true}, + } + return j +} + +func stopAllServices(c *exitControl) { + for i := range c.services { + c.services[i].Running, c.services[i].Healthy = false, false + } +} + +// 收尾只认"真的跑过恢复 + 校验",不认调用方自称已恢复。 +func TestRecoverWindowRestoresBeforeOpening(t *testing.T) { + _, c, e := instanceFixture(t) + stopAllServices(c) + if err := e.save(heldJournal("job-rec", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil { + t.Fatal(err) + } + if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil { + t.Fatal(err) + } + gate := &fakeGate{calls: &c.calls, drain: true} + if err := e.SetGate(gate); err != nil { + t.Fatal(err) + } + var restoredWith, verifiedWith string + var order []string + restorer := restorerFunc(func(_ context.Context, id string) error { + restoredWith = id + order = append(order, "restore") + return nil + }) + verifier := verifierFunc(func(_ context.Context, id string) (string, error) { + verifiedWith = id + order = append(order, "verify") + return "digest-ok", nil + }) + + if _, err := e.RecoverWindow(context.Background(), "job-rec", restorer, verifier); err != nil { + t.Fatal(err) + } + if restoredWith != strings.Repeat("b", 64) || verifiedWith != restoredWith { + t.Fatalf("restore=%q verify=%q", restoredWith, verifiedWith) + } + if strings.Join(order, ",") != "restore,verify" { + t.Fatalf("order=%v", order) + } + for _, s := range c.services { + if !s.Running || !s.Healthy { + t.Fatalf("service %s not restored after verified recovery", s.ID) + } + } + after, err := e.load("job-rec") + if err != nil || after.State != "recovered" || after.RecoveredBackup != restoredWith || + after.RecoveryDigest != "digest-ok" || after.RecoveredAt == nil { + t.Fatalf("journal=%+v err=%v", after, err) + } +} + +// 校验不通过 = 恢复未成功:服务保持停止、journal 非终态。 +func TestRecoverWindowKeepsTargetClosedWhenVerificationFails(t *testing.T) { + _, c, e := instanceFixture(t) + stopAllServices(c) + if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil { + t.Fatal(err) + } + if err := e.SetGate(&fakeGate{calls: &c.calls, drain: true}); err != nil { + t.Fatal(err) + } + if err := e.save(heldJournal("job-bad", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil { + t.Fatal(err) + } + if _, err := e.RecoverWindow(context.Background(), "job-bad", + restorerFunc(func(context.Context, string) error { return nil }), + verifierFunc(func(context.Context, string) (string, error) { + return "", errors.New("chunks 内容与预备份不一致") + })); err == nil { + t.Fatal("verification failure reported success") + } + for _, s := range c.services { + if s.Running { + t.Fatalf("service %s was started although content verification failed", s.ID) + } + } + after, err := e.load("job-bad") + if err != nil || terminal(after.State) || !after.RestoreBlocked || after.RecoveredAt != nil { + t.Fatalf("journal=%+v err=%v", after, err) + } +} + +// 没有落盘预备份身份的窗口不许"恢复":那等于凭空开放。 +func TestRecoverWindowRequiresPersistedBackup(t *testing.T) { + _, c, e := instanceFixture(t) + stopAllServices(c) + j := heldJournal("job-nobak", strings.Repeat("a", 64), "") + if err := e.save(j); err != nil { + t.Fatal(err) + } + called := false + if _, err := e.RecoverWindow(context.Background(), "job-nobak", + restorerFunc(func(context.Context, string) error { called = true; return nil }), + verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil || called { + t.Fatalf("recovery ran without a persisted backup: err=%v called=%v", err, called) + } + for _, s := range c.services { + if s.Running { + t.Fatal("writer reopened without a backup") + } + } +} + +// 目标换了(指纹不一致)就不许拿旧窗口收尾。 +func TestRecoverWindowRejectsDifferentTarget(t *testing.T) { + _, c, e := instanceFixture(t) + stopAllServices(c) + if err := e.SetTargetFingerprint(strings.Repeat("c", 64)); err != nil { + t.Fatal(err) + } + if err := e.save(heldJournal("job-other", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil { + t.Fatal(err) + } + if _, err := e.RecoverWindow(context.Background(), "job-other", + restorerFunc(func(context.Context, string) error { return nil }), + verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil { + t.Fatal("recovery accepted another target's journal") + } +} + +// 后台任务还在跑时不许开窗口(HTTP 排空覆盖不到它们)。 +func TestWindowRefusesWhenBackgroundBusy(t *testing.T) { + _, c, e := instanceFixture(t) + if err := e.SetQuiescer(&idleQuiescer{busy: true}); err != nil { + t.Fatal(err) + } + if err := e.SetQuiesceTimeout(100 * time.Millisecond); err != nil { + t.Fatal(err) + } + work := 0 + _, err := e.RunWindow(context.Background(), "job-busy", func(context.Context) error { return nil }, + func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true }) + if err == nil || work != 0 { + t.Fatalf("err=%v work=%d", err, work) + } + for _, s := range c.services { + if !s.Running { + t.Fatal("writers were stopped although the window never opened") + } + } +} + +type restorerFunc func(context.Context, string) error + +func (f restorerFunc) Restore(ctx context.Context, id string) error { return f(ctx, id) } + +type verifierFunc func(context.Context, string) (string, error) + +func (f verifierFunc) Verify(ctx context.Context, id string) (string, error) { return f(ctx, id) } diff --git a/internal/libraryfreeze/recovery.go b/internal/libraryfreeze/recovery.go new file mode 100644 index 0000000..360a91d --- /dev/null +++ b/internal/libraryfreeze/recovery.go @@ -0,0 +1,141 @@ +package libraryfreeze + +import ( + "context" + "errors" + "fmt" + "time" +) + +// Restorer 把目标内容恢复成 backupID 描述的那一份。 +// 协调器只会在"维护关闭、写入者停止"的状态下调用它,并且只传 journal 里 +// 持久化的那份预备份身份——调用方没有机会换一份。 +type Restorer interface { + Restore(ctx context.Context, backupID string) error +} + +// Verifier 逐类比对目标与 backupID 的内容(行身份 + 逐行内容 + 索引 + 资源)。 +// 返回的摘要会被持久化;返回错误即视为恢复未成功。 +type Verifier interface { + Verify(ctx context.Context, backupID string) (string, error) +} + +// RecoverWindow 是"内容风险未验证"窗口的唯一收尾方式: +// +// 绑定(原作业 + 目标指纹 + 预备份)→ 重新关入口/停写入者 → 恢复 → 真校验 +// → 持久记录校验摘要 → 恢复服务与入口 → 才允许调用方释放占用 +// +// 它不接受任何"已恢复/已验证"的入参:是否成功由这里跑出来的结果决定。 +// 任何一步失败都保持关闭(服务停着、入口关着、journal 非终态),不会放行半写目标。 +func (c *Coordinator) RecoverWindow(ctx context.Context, job string, restorer Restorer, verifier Verifier) (Result, error) { + if !validID(job) || restorer == nil || verifier == nil { + return Result{}, errors.New("invalid recovery request") + } + lock, err := c.lock() + if err != nil { + return Result{}, err + } + defer unlock(lock) + + j, err := c.load(job) + if err != nil { + return Result{}, err + } + if terminal(j.State) { + return Result{}, errors.New("freeze journal is already terminal") + } + if !j.ContentRisk { + return Result{}, errors.New("journal has no content risk; use Recover instead") + } + // 绑定:预备份身份必须已经在写目标之前落盘,目标指纹必须还是同一个。 + if !validDigest(j.BackupSnapshotID) { + return Result{}, errors.New("窗口没有落盘预备份身份,无法证明恢复目标,必须人工按归档恢复") + } + if c.fingerprint != "" && j.Level == "instance" { + if j.TargetFingerprint == "" || j.TargetFingerprint != c.fingerprint { + return Result{}, errors.New("目标身份与维护记录不一致:拒绝用旧窗口收尾另一个目标") + } + } + + // 恢复必须在维护关闭状态下做:入口先关(若已关则沿用),写入者必须停住。 + if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) { + evidence, gateErr := c.gate.CloseIngress(ctx) + if gateErr != nil { + return j.Result, fmt.Errorf("恢复前无法关闭目标入口:%w", gateErr) + } + if !evidence.Closed || !evidence.Drained { + return j.Result, errors.New("恢复前入口未确认排空") + } + j.Gate = &evidence + if err := c.save(j); err != nil { + return j.Result, err + } + } + if err := c.stopAll(ctx, &j); err != nil { + return j.Result, fmt.Errorf("恢复前无法停下写入者:%w", err) + } + if err := c.stopped(ctx, j); err != nil { + return j.Result, fmt.Errorf("恢复前写入者没有全部停住:%w", err) + } + if err := c.verifyDrain(ctx, &j); err != nil { + return j.Result, fmt.Errorf("恢复前排空未确认:%w", err) + } + if err := c.awaitBackgroundIdle(ctx, &j); err != nil { + return j.Result, fmt.Errorf("恢复前后台任务未静默:%w", err) + } + j.State = "recovering" + if err := c.save(j); err != nil { + return j.Result, err + } + + // 1) 按持久化的那份预备份恢复。 + if restoreErr := restorer.Restore(ctx, j.BackupSnapshotID); restoreErr != nil { + return c.failRecovery(&j, "restore_failed", fmt.Errorf("恢复失败:%w", restoreErr)) + } + // 2) 真校验(内容/资源/索引/Wiki),拿到摘要才算恢复成功。 + digest, verifyErr := verifier.Verify(ctx, j.BackupSnapshotID) + if verifyErr != nil { + return c.failRecovery(&j, "verify_failed", fmt.Errorf("恢复后校验未通过:%w", verifyErr)) + } + if digest == "" { + return c.failRecovery(&j, "verify_failed", errors.New("恢复后校验没有返回摘要,不能当作已验证")) + } + ended := time.Now().UTC() + j.RecoveredBackup = j.BackupSnapshotID + j.RecoveryDigest = digest + j.RecoveredAt = &ended + j.RestoreBlocked = false + if err := c.save(j); err != nil { + return j.Result, err + } + + // 3) 验证成功之后才恢复服务、才开入口。 + recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + if restoreErr := c.restore(recoveryCtx, &j); restoreErr != nil { + return c.failRecovery(&j, "service_restore_failed", fmt.Errorf("内容已恢复但服务/入口未恢复:%w", restoreErr)) + } + if c.ingressReady != nil { + if readyErr := c.ingressReady(recoveryCtx); readyErr != nil { + return c.failRecovery(&j, "ingress_unverified", fmt.Errorf("入口未恢复:%w", readyErr)) + } + } + j.State = "recovered" + j.Code = "" + j.FinishedAt = &ended + if err := c.save(j); err != nil { + return j.Result, err + } + return j.Result, nil +} + +// failRecovery 保持"目标仍不可用":服务停着、入口关着、journal 非终态。 +func (c *Coordinator) failRecovery(j *journal, code string, cause error) (Result, error) { + j.State = "held" + j.Code = code + j.RestoreBlocked = true + if err := c.save(*j); err != nil { + return j.Result, errors.Join(cause, err) + } + return j.Result, cause +} diff --git a/internal/libraryfreeze/window.go b/internal/libraryfreeze/window.go index 5808eb6..faa3319 100644 --- a/internal/libraryfreeze/window.go +++ b/internal/libraryfreeze/window.go @@ -34,6 +34,25 @@ func (w *Window) MarkContentRisk(ctx context.Context) error { return w.c.save(*w.j) } +// SetBackupSnapshot 把"这次窗口的预备份身份"落盘。恢复只认它: +// 没有这份身份的窗口,之后既不能自动恢复,也不许被外部收尾。 +func (w *Window) SetBackupSnapshot(ctx context.Context, snapshotID string) error { + if w == nil || w.c == nil || w.j == nil { + return errors.New("invalid maintenance window") + } + if err := ctx.Err(); err != nil { + return err + } + if !validDigest(snapshotID) { + return errors.New("invalid backup snapshot id") + } + if w.j.BackupSnapshotID != "" && w.j.BackupSnapshotID != snapshotID { + return errors.New("backup snapshot changed inside the same window") + } + w.j.BackupSnapshotID = snapshotID + return w.c.save(*w.j) +} + // Job 返回窗口所属作业 id(恢复器与调用方据此对账)。 func (w *Window) Job() string { if w == nil || w.j == nil { @@ -113,7 +132,7 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co return Result{}, err } j = journal{Version: 1, Source: c.source, Job: job, Level: c.level, Ingress: c.ingress, - Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()} + TargetFingerprint: c.fingerprint, Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()} for _, s := range items { if s.Running && !s.Healthy { return Result{}, errors.New("test writer is not ready") @@ -125,6 +144,12 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co } } + // 后台任务必须先在"还能跑"的时候跑完:入口排空只覆盖 HTTP 连接, + // app 内的 asynq worker 会被上游的 exit(1) 直接切断。 + if idleErr := c.awaitBackgroundIdle(ctx, &j); idleErr != nil { + return c.abandonWindow(ctx, &j, idleErr, "background_busy") + } + // 顺序是有原因的:入口先停(新请求被拒、在途请求跑完),写者后停。 // 反过来做,正在处理的请求就会被 SIGTERM 切断——那不算排空。 if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) { @@ -267,6 +292,42 @@ func (c *Coordinator) closeWindow(ctx context.Context, j *journal, cause error, return j.Result, cause } +// awaitBackgroundIdle 等到后台任务队列为空再往下走。拿不到结论(查询失败) +// 也按"没空"处理:不能把"问不到"当成"没任务"。 +func (c *Coordinator) awaitBackgroundIdle(ctx context.Context, j *journal) error { + if c.quiescer == nil { + if c.level == "instance" { + return errors.New("没有后台任务静默检查:不能证明停写者时后台任务已结束") + } + return nil + } + wait := c.quiesceWait + if wait <= 0 { + wait = 2 * time.Minute + } + deadline := time.Now().Add(wait) + var last string + for { + idle, detail, err := c.quiescer.Idle(ctx) + if err != nil { + return fmt.Errorf("读不到后台任务状态:%w", err) + } + if idle { + j.BackgroundIdle = true + return nil + } + last = detail + if time.Now().After(deadline) { + return fmt.Errorf("后台任务仍在运行(%s):等它结束再开窗口,否则会被 SIGTERM 切断", last) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } +} + // stopAll 只对"原本在跑"的服务发停止指令,每发一个都先落盘意图: // 进程在停止过程中死掉时,磁盘上仍能看出"哪些服务被要求停过"。 func (c *Coordinator) stopAll(ctx context.Context, j *journal) error { @@ -342,63 +403,3 @@ func validDigest(id string) bool { decoded, err := hex.DecodeString(id) return err == nil && len(decoded) == 32 && strings.ToLower(id) == id } - -// FinalizeAfterOperatorRecovery 收尾一个"内容风险未验证 + 人工已恢复"的窗口。 -// -// 它不启动任何服务、也不释放治理库存的占用:只做两件事——核对"原本在跑的服务 -// 现在确实在跑且健康、入口确实在服务",然后把 journal 标成 operator_recovered。 -// 没有这两条核对,它就退化成"把卡住的记录清掉",那是另一种误放行。 -func (c *Coordinator) FinalizeAfterOperatorRecovery(ctx context.Context, job string) error { - if !validID(job) { - return errors.New("invalid freeze job id") - } - lock, err := c.lock() - if err != nil { - return err - } - defer unlock(lock) - j, err := c.load(job) - if err != nil { - return err - } - if terminal(j.State) { - return errors.New("freeze journal is already terminal") - } - if !j.ContentRisk { - return errors.New("journal has no content risk; use Recover instead") - } - items, err := inventory(ctx, c.control) - if err != nil { - return err - } - running := map[string]Service{} - for _, item := range items { - running[item.ID] = item - } - for _, s := range j.Services { - if !s.Running { - continue - } - now, ok := running[s.ID] - if !ok || !now.Running || !now.Healthy { - return fmt.Errorf("服务 %s 还没有恢复到健康状态,不能收尾这个窗口", s.ID) - } - } - if c.gate != nil && j.Gate != nil && !j.GateReopened { - if err := c.gate.OpenIngress(ctx); err != nil { - return fmt.Errorf("入口还没有恢复:%w", err) - } - j.GateReopened = true - } - if c.ingressReady != nil { - if err := c.ingressReady(ctx); err != nil { - return fmt.Errorf("入口验证未通过:%w", err) - } - } - ended := time.Now().UTC() - j.State = "failed" - j.Code = "operator_recovered" - j.RestoreBlocked = false - j.FinishedAt = &ended - return c.save(j) -} diff --git a/internal/libraryfreeze/window_test.go b/internal/libraryfreeze/window_test.go index 495b971..a05465e 100644 --- a/internal/libraryfreeze/window_test.go +++ b/internal/libraryfreeze/window_test.go @@ -29,6 +29,16 @@ func (c *exitControl) Inventory(ctx context.Context) ([]Service, error) { return items, nil } +// idleQuiescer:实例级窗口必须能证明后台任务已空,测试里默认给一个空的。 +type idleQuiescer struct{ busy bool } + +func (q *idleQuiescer) Idle(context.Context) (bool, string, error) { + if q.busy { + return false, "asynq pending=1", nil + } + return true, "asynq=0 pending_ops=0", nil +} + func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) { t.Helper() root := t.TempDir() @@ -45,6 +55,9 @@ func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) { if err := engine.SetTarget("instance", "http://127.0.0.1:1/health", nil, nil); err != nil { t.Fatal(err) } + if err := engine.SetQuiescer(&idleQuiescer{}); err != nil { + t.Fatal(err) + } return root, c, engine } @@ -312,32 +325,3 @@ func TestWindowRefusesWhenGateNotDrained(t *testing.T) { t.Fatalf("writers were stopped before the ingress was drained: %v", c.calls) } } - -// 人工恢复后的收尾:服务没起来就必须拒绝(不能靠清记录放行)。 -func TestFinalizeAfterOperatorRecoveryRequiresHealthyServices(t *testing.T) { - _, c, e := instanceFixture(t) - _, err := e.RunWindow(context.Background(), "job-manual", func(context.Context) error { return nil }, - func(ctx context.Context, w *Window) (string, error) { - if err := w.MarkContentRisk(ctx); err != nil { - t.Fatal(err) - } - return "", errors.New("内容回滚失败") - }, func(error) bool { return false }) - if err == nil { - t.Fatal("blocked window reported success") - } - // 服务还停着:收尾必须失败。 - if err := e.FinalizeAfterOperatorRecovery(context.Background(), "job-manual"); err == nil { - t.Fatal("finalize accepted a still-stopped target") - } - for i := range c.services { - c.services[i].Running, c.services[i].Healthy = true, true - } - if err := e.FinalizeAfterOperatorRecovery(context.Background(), "job-manual"); err != nil { - t.Fatalf("finalize rejected a recovered target: %v", err) - } - j, err := e.load("job-manual") - if err != nil || j.State != "failed" || j.Code != "operator_recovered" || j.RestoreBlocked { - t.Fatalf("journal=%+v err=%v", j, err) - } -} diff --git a/internal/librarymaintenance/dockerapi.go b/internal/librarymaintenance/dockerapi.go new file mode 100644 index 0000000..88af4c6 --- /dev/null +++ b/internal/librarymaintenance/dockerapi.go @@ -0,0 +1,141 @@ +package librarymaintenance + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "time" +) + +// dockerAPI 是目标维护用到的 Docker Engine 最小客户端(unix socket)。 +// 入口闸门与拓扑校验共用同一份实现:两处各自拼一套请求只会长出两套毛病。 +type dockerAPI struct { + client *http.Client +} + +func newDockerAPI(socket string, timeout time.Duration) (*dockerAPI, error) { + if socket == "" || timeout <= 0 { + return nil, errors.New("invalid docker api configuration") + } + info, err := os.Stat(socket) + if err != nil || info.Mode()&os.ModeSocket == 0 { + return nil, errors.New("local Docker socket unavailable") + } + transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socket) + }} + return &dockerAPI{client: &http.Client{Transport: transport, Timeout: timeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("docker redirects prohibited") }}}, nil +} + +func (d *dockerAPI) Close() { d.client.CloseIdleConnections() } + +func (d *dockerAPI) request(ctx context.Context, method, path string, out any) error { + req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil) + if err != nil { + return errors.New("invalid docker request") + } + resp, err := d.client.Do(req) + if err != nil { + return errors.New("docker request failed") + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("docker request rejected: %d", resp.StatusCode) + } + if out != nil { + data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if readErr != nil || json.Unmarshal(data, out) != nil { + return errors.New("invalid docker response") + } + } + return nil +} + +type containerSummary struct { + ID string `json:"Id"` + Labels map[string]string `json:"Labels"` + Ports []struct { + PrivatePort int `json:"PrivatePort"` + PublicPort int `json:"PublicPort"` + Type string `json:"Type"` + } `json:"Ports"` +} + +func (d *dockerAPI) listProject(ctx context.Context, project string) ([]containerSummary, error) { + filters, _ := json.Marshal(map[string][]string{"label": {"com.docker.compose.project=" + project}}) + var items []containerSummary + if err := d.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &items); err != nil { + return nil, err + } + return items, nil +} + +type containerState struct { + ID string `json:"Id"` + Labels map[string]string `json:"Config"` + Config struct { + Labels map[string]string `json:"Labels"` + } `json:"-"` + State struct { + Running bool + Status string + ExitCode int + Health *struct{ Status string } + } `json:"State"` + HostConfig struct { + PortBindings map[string][]struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` + } `json:"PortBindings"` + } `json:"HostConfig"` + NetworkSettings struct { + Ports map[string][]struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` + } `json:"Ports"` + } `json:"NetworkSettings"` +} + +func (d *dockerAPI) inspect(ctx context.Context, id string) (containerState, error) { + var data containerState + if err := d.request(ctx, "GET", "/containers/"+id+"/json", &data); err != nil { + return containerState{}, err + } + return data, nil +} + +// publishedPorts 返回容器对外发布的端口(HostConfig + NetworkSettings 都算)。 +// 写入者只要发布了一个端口,就存在"绕开入口闸门直连"的通道。 +func (c containerState) publishedPorts() []string { + seen := map[string]bool{} + out := make([]string, 0, 2) + add := func(hostIP, hostPort, containerPort string) { + if hostPort == "" || hostPort == "0" { + return + } + key := hostIP + ":" + hostPort + "->" + containerPort + if seen[key] { + return + } + seen[key] = true + out = append(out, key) + } + for port, bindings := range c.HostConfig.PortBindings { + for _, binding := range bindings { + add(binding.HostIP, binding.HostPort, port) + } + } + for port, bindings := range c.NetworkSettings.Ports { + for _, binding := range bindings { + add(binding.HostIP, binding.HostPort, port) + } + } + return out +} diff --git a/internal/librarymaintenance/gate.go b/internal/librarymaintenance/gate.go index 5512488..7242acd 100644 --- a/internal/librarymaintenance/gate.go +++ b/internal/librarymaintenance/gate.go @@ -6,11 +6,7 @@ import ( "errors" "fmt" "io" - "net" - "net/http" - "net/url" "os" - "path/filepath" "time" "knowledge-governance/internal/libraryfreeze" @@ -20,8 +16,7 @@ import ( // 闸门自己写下的 drain.json(停止接客时还有多少在途连接、最后是否归零)。 // 不看 HTTP 探测就宣布"入口关了"——那是猜。 type dockerGate struct { - client *http.Client - socket string + api *dockerAPI project string service string drainFile string @@ -29,79 +24,41 @@ type dockerGate struct { } func newDockerGate(socket, project, service, drainFile string, timeout time.Duration) (*dockerGate, error) { - if !filepath.IsAbs(socket) || project == "" || service == "" || !filepath.IsAbs(drainFile) { + if project == "" || service == "" || drainFile == "" || drainFile[0] != '/' { return nil, errors.New("入口闸门配置不完整") } - info, err := os.Stat(socket) - if err != nil || info.Mode()&os.ModeSocket == 0 { - return nil, errors.New("local Docker socket unavailable") + api, err := newDockerAPI(socket, timeout) + if err != nil { + return nil, err } - transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, "unix", socket) - }} - return &dockerGate{ - client: &http.Client{Transport: transport, Timeout: timeout, - CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("docker redirects prohibited") }}, - socket: socket, project: project, service: service, drainFile: drainFile, timeout: timeout, - }, nil + return &dockerGate{api: api, project: project, service: service, drainFile: drainFile, timeout: timeout}, nil } -func (g *dockerGate) request(ctx context.Context, method, path string, out any) error { - req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil) +func (g *dockerGate) container(ctx context.Context) (containerState, error) { + items, err := g.api.listProject(ctx, g.project) if err != nil { - return errors.New("invalid docker request") + return containerState{}, err } - resp, err := g.client.Do(req) - if err != nil { - return errors.New("docker request failed") - } - defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return fmt.Errorf("docker request rejected: %d", resp.StatusCode) - } - if out != nil { - data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - if readErr != nil || json.Unmarshal(data, out) != nil { - return errors.New("invalid docker response") + id := "" + for _, item := range items { + if item.Labels["com.docker.compose.service"] != g.service { + continue } + if id != "" { + return containerState{}, fmt.Errorf("入口闸门实例数异常:多于一个 %s", g.service) + } + id = item.ID } - return nil -} - -type gateInspection struct { - ID string `json:"Id"` - Config struct { - Labels map[string]string `json:"Labels"` - } `json:"Config"` - State struct { - Running bool - Status string - ExitCode int - Health *struct{ Status string } - } `json:"State"` -} - -func (g *dockerGate) container(ctx context.Context) (gateInspection, error) { - filters, _ := json.Marshal(map[string][]string{"label": { - "com.docker.compose.project=" + g.project, - "com.docker.compose.service=" + g.service, - }}) - var listed []struct { - ID string `json:"Id"` + if id == "" { + return containerState{}, fmt.Errorf("找不到入口闸门服务 %s", g.service) } - if err := g.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &listed); err != nil { - return gateInspection{}, err - } - if len(listed) != 1 { - return gateInspection{}, fmt.Errorf("入口闸门实例数异常:%d", len(listed)) - } - var data gateInspection - if err := g.request(ctx, "GET", "/containers/"+listed[0].ID+"/json", &data); err != nil { - return gateInspection{}, err + data, err := g.api.inspect(ctx, id) + if err != nil { + return containerState{}, err } if data.Config.Labels["com.docker.compose.project"] != g.project || data.Config.Labels["com.docker.compose.service"] != g.service { - return gateInspection{}, errors.New("入口闸门身份不匹配") + return containerState{}, errors.New("入口闸门身份不匹配") } return data, nil } @@ -151,7 +108,7 @@ func (g *dockerGate) CloseIngress(ctx context.Context) (libraryfreeze.GateEviden } else if !errors.Is(err, os.ErrNotExist) { return libraryfreeze.GateEvidence{}, fmt.Errorf("无法确认排空记录已清理:%w", err) } - if err := g.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil { + if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil { return libraryfreeze.GateEvidence{}, err } deadline := time.Now().Add(g.timeout) @@ -202,7 +159,7 @@ func (g *dockerGate) OpenIngress(ctx context.Context) error { return err } if !data.State.Running { - if err := g.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil { + if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil { return err } } diff --git a/internal/librarymaintenance/quiesce.go b/internal/librarymaintenance/quiesce.go new file mode 100644 index 0000000..c78e010 --- /dev/null +++ b/internal/librarymaintenance/quiesce.go @@ -0,0 +1,248 @@ +package librarymaintenance + +import ( + "bufio" + "context" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// targetQuiescer 回答"后台写入者是不是已经空了": +// - 目标库:未结清的后台操作、未完成的解析(这两张表就是 app 的后台账本); +// - 目标队列:asynq 里还在排队/在跑/待重试的任务。 +// +// 为什么要单独做:HTTP 入口排空只覆盖连接;app 内的 asynq worker 会被上游的 +// SIGTERM exit(1) 切断,正在跑的任务就这样被腰斩。窗口必须先等它们结束。 +type targetQuiescer struct { + pool *pgxpool.Pool + redisAddr string + redisPass string + redisDB int + timeout time.Duration + queueMatch string +} + +func newTargetQuiescer(ctx context.Context, databaseURL, redisAddr, redisPass string, redisDB int, timeout time.Duration) (*targetQuiescer, error) { + if databaseURL == "" || redisAddr == "" { + return nil, errors.New("后台静默检查缺少目标库或队列地址") + } + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + return nil, errors.New("后台静默检查连不上目标库") + } + if timeout <= 0 { + timeout = 10 * time.Second + } + return &targetQuiescer{pool: pool, redisAddr: redisAddr, redisPass: redisPass, redisDB: redisDB, + timeout: timeout, queueMatch: "asynq:*"}, nil +} + +func (q *targetQuiescer) Close() { q.pool.Close() } + +func (q *targetQuiescer) Idle(ctx context.Context) (bool, string, error) { + var pendingOps, unsettled int64 + // task_pending_ops / knowledges 是 app 自己的后台账本:不看它们只看 HTTP 会漏。 + if err := q.pool.QueryRow(ctx, `SELECT + (SELECT count(*) FROM task_pending_ops), + (SELECT count(*) FROM knowledges WHERE deleted_at IS NULL + AND (parse_status IS DISTINCT FROM 'completed' OR pending_subtasks_count IS NULL OR pending_subtasks_count<>0))`). + Scan(&pendingOps, &unsettled); err != nil { + return false, "", fmt.Errorf("读不到后台账本:%w", err) + } + depth, err := q.queueDepth(ctx) + if err != nil { + return false, "", err + } + detail := fmt.Sprintf("pending_ops=%d unsettled_knowledges=%d asynq_active=%d", pendingOps, unsettled, depth) + return pendingOps == 0 && unsettled == 0 && depth == 0, detail, nil +} + +// queueDepth 统计 asynq 里"还没结束"的任务数(排队/在跑/待重试)。 +// archived/completed 是历史,不算忙。 +func (q *targetQuiescer) queueDepth(ctx context.Context) (int64, error) { + conn, err := dialRedis(q.redisAddr, q.redisPass, q.redisDB, q.timeout) + if err != nil { + return 0, fmt.Errorf("连不上目标队列:%w", err) + } + defer conn.Close() + keys, err := conn.scan(ctx, q.queueMatch) + if err != nil { + return 0, fmt.Errorf("扫描目标队列失败:%w", err) + } + var total int64 + for _, key := range keys { + suffix := key[strings.LastIndex(key, ":"):] + switch suffix { + case ":pending", ":active", ":inprogress": + n, err := conn.intCmd(ctx, "LLEN", key) + if err != nil { + return 0, err + } + total += n + case ":scheduled", ":retry": + n, err := conn.intCmd(ctx, "ZCARD", key) + if err != nil { + return 0, err + } + total += n + } + } + return total, nil +} + +// ---- 最小 RESP 客户端:只用到 AUTH/SELECT/SCAN/LLEN/ZCARD,不引第三方库 ---- + +type redisConn struct { + conn net.Conn + reader *bufio.Reader + timeout time.Duration +} + +func dialRedis(addr, password string, db int, timeout time.Duration) (*redisConn, error) { + dialer := &net.Dialer{Timeout: timeout} + conn, err := dialer.Dial("tcp", addr) + if err != nil { + return nil, err + } + _ = conn.SetDeadline(time.Now().Add(timeout)) + c := &redisConn{conn: conn, reader: bufio.NewReader(conn), timeout: timeout} + if password != "" { + if _, err := c.do("AUTH", password); err != nil { + conn.Close() + return nil, err + } + } + if db != 0 { + if _, err := c.do("SELECT", strconv.Itoa(db)); err != nil { + conn.Close() + return nil, err + } + } + return c, nil +} + +func (c *redisConn) Close() error { return c.conn.Close() } + +func (c *redisConn) do(args ...string) (any, error) { + _ = c.conn.SetDeadline(time.Now().Add(c.timeout)) + var builder strings.Builder + builder.WriteString("*" + strconv.Itoa(len(args)) + "\r\n") + for _, arg := range args { + builder.WriteString("$" + strconv.Itoa(len(arg)) + "\r\n" + arg + "\r\n") + } + if _, err := c.conn.Write([]byte(builder.String())); err != nil { + return nil, err + } + return c.read() +} + +func (c *redisConn) read() (any, error) { + line, err := c.reader.ReadString('\n') + if err != nil { + return nil, err + } + line = strings.TrimRight(line, "\r\n") + if line == "" { + return nil, errors.New("empty redis reply") + } + switch line[0] { + case '+': + return line[1:], nil + case '-': + return nil, errors.New(line[1:]) + case ':': + value, err := strconv.ParseInt(line[1:], 10, 64) + return value, err + case '$': + length, err := strconv.Atoi(line[1:]) + if err != nil { + return nil, err + } + if length < 0 { + return nil, nil + } + buf := make([]byte, length+2) + if _, err := ioReadFull(c.reader, buf); err != nil { + return nil, err + } + return string(buf[:length]), nil + case '*': + count, err := strconv.Atoi(line[1:]) + if err != nil { + return nil, err + } + items := make([]any, 0, count) + for i := 0; i < count; i++ { + item, err := c.read() + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil + default: + return nil, errors.New("unsupported redis reply") + } +} + +func ioReadFull(reader *bufio.Reader, buf []byte) (int, error) { + total := 0 + for total < len(buf) { + n, err := reader.Read(buf[total:]) + total += n + if err != nil { + return total, err + } + } + return total, nil +} + +func (c *redisConn) scan(ctx context.Context, match string) ([]string, error) { + var keys []string + cursor := "0" + for { + if err := ctx.Err(); err != nil { + return nil, err + } + reply, err := c.do("SCAN", cursor, "MATCH", match, "COUNT", "1000") + if err != nil { + return nil, err + } + items, ok := reply.([]any) + if !ok || len(items) != 2 { + return nil, errors.New("unexpected SCAN reply") + } + next, _ := items[0].(string) + batch, _ := items[1].([]any) + for _, item := range batch { + if key, ok := item.(string); ok { + keys = append(keys, key) + } + } + if next == "0" || next == "" { + return keys, nil + } + cursor = next + } +} + +func (c *redisConn) intCmd(ctx context.Context, cmd string, args ...string) (int64, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + reply, err := c.do(append([]string{cmd}, args...)...) + if err != nil { + return 0, err + } + value, ok := reply.(int64) + if !ok { + return 0, errors.New("unexpected integer reply") + } + return value, nil +} diff --git a/internal/librarymaintenance/target.go b/internal/librarymaintenance/target.go index 3ce40e1..33aa899 100644 --- a/internal/librarymaintenance/target.go +++ b/internal/librarymaintenance/target.go @@ -56,7 +56,12 @@ type Config struct { Database string Index string Bucket string - Timeout time.Duration + // 后台静默检查需要的运行时依赖(不写进维护记录)。 + DatabaseURL string + RedisAddr string + RedisPassword string + RedisDB int + Timeout time.Duration } type Manager struct { @@ -68,6 +73,18 @@ type Manager struct { timeout time.Duration client *http.Client gate *dockerGate + api *dockerAPI + quiescer *targetQuiescer +} + +// Close 释放拓扑/静默检查持有的连接。 +func (m *Manager) Close() { + if m.api != nil { + m.api.Close() + } + if m.quiescer != nil { + m.quiescer.Close() + } } func New(cfg Config) (*Manager, error) { @@ -102,9 +119,13 @@ func New(cfg Config) (*Manager, error) { return nil, err } target.Fingerprint = fingerprint + api, err := newDockerAPI(cfg.DockerSocket, cfg.Timeout) + if err != nil { + return nil, err + } manager := &Manager{target: target, source: project, docker: cfg.DockerSocket, writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout, - client: &http.Client{Timeout: 5 * time.Second}} + client: &http.Client{Timeout: 5 * time.Second}, api: api} if cfg.IngressService != "" || cfg.IngressDrainFile != "" { gate, err := newDockerGate(cfg.DockerSocket, project, cfg.IngressService, cfg.IngressDrainFile, cfg.Timeout) if err != nil { @@ -112,6 +133,14 @@ func New(cfg Config) (*Manager, error) { } manager.gate = gate } + if cfg.DatabaseURL != "" && cfg.RedisAddr != "" { + quiescer, err := newTargetQuiescer(context.Background(), cfg.DatabaseURL, cfg.RedisAddr, + cfg.RedisPassword, cfg.RedisDB, cfg.Timeout) + if err != nil { + return nil, err + } + manager.quiescer = quiescer + } return manager, nil } @@ -159,6 +188,11 @@ func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze. control.Close() return nil, nil, err } + if err := coordinator.SetTargetFingerprint(m.target.Fingerprint); err != nil { + coordinator.Close() + control.Close() + return nil, nil, err + } if m.gate != nil { if err := coordinator.SetGate(m.gate); err != nil { coordinator.Close() @@ -166,6 +200,17 @@ func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze. return nil, nil, err } } + // 后台静默检查是实例级窗口的必备条件:没有它就等于只排空了 HTTP。 + if m.quiescer == nil { + coordinator.Close() + control.Close() + return nil, nil, errors.New("缺少后台静默检查(目标库/队列未配置),拒绝开实例级窗口") + } + if err := coordinator.SetQuiescer(m.quiescer); err != nil { + coordinator.Close() + control.Close() + return nil, nil, err + } return coordinator, control, nil } @@ -270,3 +315,82 @@ func DatabaseIdentity(raw string) (string, error) { } return parsed.Host + "/" + database, nil } + +const composeServiceLabel = "com.docker.compose.service" + +// VerifyTopology 强制"入口唯一": +// - 写入者不许发布任何宿主机端口(否则就存在绕闸直连,维护窗口覆盖不到); +// - 入口闸门必须存在、必须是独立服务,并且发布了入口 URL 用的那个端口。 +// +// 任何一条不满足都拒绝开窗口——拓扑错了,"维护"就只是关了一半的门。 +func (m *Manager) VerifyTopology(ctx context.Context) error { + items, err := m.api.listProject(ctx, m.target.Project) + if err != nil { + return err + } + byService := make(map[string]containerSummary, len(items)) + for _, item := range items { + name := item.Labels[composeServiceLabel] + if name == "" { + return errors.New("项目里有容器缺少 compose service 标签,无法确认写入者边界") + } + if _, duplicated := byService[name]; duplicated { + return fmt.Errorf("项目里服务 %s 有多个实例,无法确认写入者边界", name) + } + byService[name] = item + } + for _, writer := range m.writers { + item, ok := byService[writer] + if !ok { + return fmt.Errorf("声明的写入者 %s 不存在", writer) + } + state, err := m.api.inspect(ctx, item.ID) + if err != nil { + return err + } + if ports := state.publishedPorts(); len(ports) > 0 { + return fmt.Errorf("写入者 %s 直接发布了端口 %s:存在绕开入口闸门的直连通道,拒绝开维护窗口", + writer, strings.Join(ports, ",")) + } + } + if m.gate == nil { + return errors.New("实例级维护必须声明入口闸门") + } + for _, writer := range m.writers { + if writer == m.gate.service { + return errors.New("入口闸门不能同时被声明为写入者") + } + } + gateItem, ok := byService[m.gate.service] + if !ok { + return fmt.Errorf("声明的入口闸门 %s 不存在", m.gate.service) + } + gateState, err := m.api.inspect(ctx, gateItem.ID) + if err != nil { + return err + } + published := gateState.publishedPorts() + if len(published) == 0 { + return fmt.Errorf("入口闸门 %s 没有发布任何端口:外部无法访问,入口定义不成立", m.gate.service) + } + // 入口 URL 的端口必须真的是这个闸门发布的端口。 + parsed, err := url.Parse(m.target.Ingress) + if err != nil || parsed.Host == "" { + return errors.New("目标入口地址无效") + } + hostPort := parsed.Port() + if hostPort == "" { + if parsed.Scheme == "https" { + hostPort = "443" + } else { + hostPort = "80" + } + } + for _, entry := range published { + if strings.Contains(entry, ":"+hostPort+"->") { + return nil + } + } + return fmt.Errorf("入口 %s 的端口 %s 不是闸门发布的端口(%s):入口指向了别的服务", + m.target.Ingress, hostPort, strings.Join(published, ",")) +} diff --git a/internal/librarymaintenance/target_test.go b/internal/librarymaintenance/target_test.go index 69ad5b2..547f062 100644 --- a/internal/librarymaintenance/target_test.go +++ b/internal/librarymaintenance/target_test.go @@ -132,3 +132,24 @@ func TestDatabaseIdentityDropsCredentials(t *testing.T) { t.Fatal("invalid DSN accepted") } } + +// 写入者只要发布了宿主机端口,就存在绕开闸门的直连通道。 +func TestPublishedPortsDetectsDirectBypass(t *testing.T) { + var state containerState + state.HostConfig.PortBindings = map[string][]struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` + }{"8080/tcp": {{HostIP: "127.0.0.1", HostPort: "18081"}}} + ports := state.publishedPorts() + if len(ports) != 1 || !strings.Contains(ports[0], "18081") { + t.Fatalf("publishedPorts=%v", ports) + } + var bare containerState + bare.HostConfig.PortBindings = map[string][]struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` + }{"8080/tcp": nil} + if got := bare.publishedPorts(); len(got) != 0 { + t.Fatalf("unpublished container reported ports: %v", got) + } +} diff --git a/internal/platform/postgres/library_instance_maintenance.go b/internal/platform/postgres/library_instance_maintenance.go new file mode 100644 index 0000000..6a6b468 --- /dev/null +++ b/internal/platform/postgres/library_instance_maintenance.go @@ -0,0 +1,177 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" +) + +// LibraryInstanceMaintenance 是"实例级维护"的占用记录:一个目标实例同一时刻 +// 只允许一个窗口,无论它来自 internal 还是 external。 +// +// 为什么不是 scope 锁:stop/start 的是同一批容器,两个 scope 各持一个 scope 锁 +// 时互相看不见,会同时停/起同一实例——那是假互斥,比没有锁更危险。 +type LibraryInstanceMaintenance struct { + Instance string `json:"instance"` + OwnerJobID string `json:"owner_job_id"` + OwnerScope string `json:"owner_scope"` + TargetFingerprint string `json:"target_fingerprint"` + Level string `json:"level"` + Services []string `json:"services"` + IngressState string `json:"ingress_state"` + State string `json:"state"` + OriginalServiceState string `json:"original_service_state"` + Note string `json:"note"` + AcquiredAt time.Time `json:"acquired_at"` + UpdatedAt time.Time `json:"updated_at"` + ReleasedAt *time.Time `json:"released_at,omitempty"` +} + +const instanceMaintenanceColumns = `instance, owner_job_id, owner_scope, target_fingerprint, level, services, + ingress_state, state, original_service_state, note, acquired_at, updated_at, released_at` + +// AcquireInstanceMaintenance 原子地拿到实例维护权: +// 已释放的记录可以被新作业接管;仍被持有的只允许原作业重入。 +func (s *Store) AcquireInstanceMaintenance(ctx context.Context, item LibraryInstanceMaintenance, now time.Time) (LibraryInstanceMaintenance, error) { + level := item.Level + if level == "" { + level = "instance" + } + ingress := item.IngressState + if ingress == "" { + ingress = "unknown" + } + services := item.Services + if services == nil { + services = []string{} + } + raw, err := json.Marshal(services) + if err != nil { + return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err) + } + row := s.db.QueryRow(ctx, ` +INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level, services, + ingress_state, state, original_service_state, note, acquired_at, updated_at) +VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$11) +ON CONFLICT (instance) DO UPDATE SET + owner_job_id = EXCLUDED.owner_job_id, + owner_scope = EXCLUDED.owner_scope, + target_fingerprint = EXCLUDED.target_fingerprint, + level = EXCLUDED.level, + services = EXCLUDED.services, + ingress_state = EXCLUDED.ingress_state, + state = EXCLUDED.state, + original_service_state = EXCLUDED.original_service_state, + note = EXCLUDED.note, + updated_at = EXCLUDED.updated_at, + released_at = NULL +WHERE library_instance_maintenance.released_at IS NOT NULL + OR library_instance_maintenance.owner_job_id = EXCLUDED.owner_job_id +RETURNING `+instanceMaintenanceColumns, + item.Instance, item.OwnerJobID, item.OwnerScope, item.TargetFingerprint, level, string(raw), + ingress, item.State, item.OriginalServiceState, item.Note, now.UTC()) + acquired, err := scanInstanceMaintenance(row) + if errors.Is(err, pgx.ErrNoRows) { + return LibraryInstanceMaintenance{}, ErrMaintenanceHeld + } + if err != nil { + return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err) + } + return acquired, nil +} + +// UpdateInstanceMaintenanceState 只允许持有者改自己的窗口状态。 +func (s *Store) UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error { + tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state=$3, note=$4, updated_at=$5 +WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC()) + if err != nil { + return fmt.Errorf("update instance maintenance: %w", err) + } + if tag.RowsAffected() == 0 { + return errors.New("instance maintenance not held by this job") + } + return nil +} + +// SetInstanceMaintenanceIngress 记录入口真实状态。 +func (s *Store) SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error { + if state != "closed" && state != "open" { + return errors.New("invalid ingress state") + } + tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET ingress_state=$3, note=$4, updated_at=$5 +WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC()) + if err != nil { + return fmt.Errorf("update instance maintenance ingress: %w", err) + } + if tag.RowsAffected() == 0 { + return errors.New("instance maintenance not held by this job") + } + return nil +} + +// ReleaseInstanceMaintenance 只有"内容已验证 + 服务与入口恢复"才允许调用。 +func (s *Store) ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error { + tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state='released', released_at=$3, updated_at=$3 +WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, now.UTC()) + if err != nil { + return fmt.Errorf("release instance maintenance: %w", err) + } + if tag.RowsAffected() == 0 { + return errors.New("instance maintenance not held by this job") + } + return nil +} + +// InstanceMaintenanceOf 读当前实例窗口(接口互斥判断与页面展示都用它)。 +func (s *Store) InstanceMaintenanceOf(ctx context.Context, instance string) (*LibraryInstanceMaintenance, error) { + row := s.db.QueryRow(ctx, `SELECT `+instanceMaintenanceColumns+` FROM library_instance_maintenance WHERE instance=$1`, instance) + item, err := scanInstanceMaintenance(row) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +// HeldInstanceMaintenance 返回仍然持有(未释放)的实例窗口,供接口在不知道 +// 具体实例名时也拒绝写入。 +func (s *Store) HeldInstanceMaintenance(ctx context.Context) ([]LibraryInstanceMaintenance, error) { + rows, err := s.db.Query(ctx, `SELECT `+instanceMaintenanceColumns+` +FROM library_instance_maintenance WHERE released_at IS NULL ORDER BY instance`) + if err != nil { + return nil, fmt.Errorf("list instance maintenance: %w", err) + } + defer rows.Close() + items := make([]LibraryInstanceMaintenance, 0, 2) + for rows.Next() { + item, err := scanInstanceMaintenance(rows) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, rows.Err() +} + +func scanInstanceMaintenance(row pgx.Row) (LibraryInstanceMaintenance, error) { + var item LibraryInstanceMaintenance + var raw []byte + err := row.Scan(&item.Instance, &item.OwnerJobID, &item.OwnerScope, &item.TargetFingerprint, &item.Level, + &raw, &item.IngressState, &item.State, &item.OriginalServiceState, &item.Note, + &item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt) + if err != nil { + return LibraryInstanceMaintenance{}, err + } + if len(raw) > 0 { + if err := json.Unmarshal(raw, &item.Services); err != nil { + return LibraryInstanceMaintenance{}, fmt.Errorf("instance maintenance services unreadable: %w", err) + } + } + return item, nil +} diff --git a/internal/platform/postgres/library_maintenance_integration_test.go b/internal/platform/postgres/library_maintenance_integration_test.go index 9a036c4..1dc779c 100644 --- a/internal/platform/postgres/library_maintenance_integration_test.go +++ b/internal/platform/postgres/library_maintenance_integration_test.go @@ -71,3 +71,45 @@ func TestLibraryMaintenanceOwnershipAcrossRelease(t *testing.T) { t.Fatalf("released window not re-acquired cleanly: %+v", second) } } + +// 实例级占用必须跨 scope 互斥:internal 持有期间 external 也进不来。 +// (旧实现按 scope 建行,两个 scope 各持一个锁,会同时停/起同一个实例。) +func TestInstanceMaintenanceExcludesAcrossScopes(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("migrate: %v", err) + } + store := NewStore(pool) + now := time.Now().UTC() + base := LibraryInstanceMaintenance{Instance: "kgmaint-test", OwnerJobID: "job-internal", + OwnerScope: "internal", TargetFingerprint: "fp", Level: "instance", Services: []string{"app"}, + State: "acquired", Note: "test"} + if _, err := store.AcquireInstanceMaintenance(ctx, base, now); err != nil { + t.Fatalf("first acquire: %v", err) + } + external := base + external.OwnerJobID = "job-external" + external.OwnerScope = "external" + if _, err := store.AcquireInstanceMaintenance(ctx, external, now.Add(time.Second)); !errors.Is(err, ErrMaintenanceHeld) { + t.Fatalf("external scope entered while internal held the instance: err=%v", err) + } + if err := store.ReleaseInstanceMaintenance(ctx, "kgmaint-test", "job-internal", now.Add(2*time.Second)); err != nil { + t.Fatalf("release: %v", err) + } + if _, err := store.AcquireInstanceMaintenance(ctx, external, now.Add(3*time.Second)); err != nil { + t.Fatalf("external scope could not take over the released instance: %v", err) + } + held, err := store.HeldInstanceMaintenance(ctx) + if err != nil || len(held) != 1 || held[0].OwnerScope != "external" { + t.Fatalf("held=%+v err=%v", held, err) + } +} diff --git a/internal/platform/postgres/migrations/000072_library_instance_maintenance.down.sql b/internal/platform/postgres/migrations/000072_library_instance_maintenance.down.sql new file mode 100644 index 0000000..443f529 --- /dev/null +++ b/internal/platform/postgres/migrations/000072_library_instance_maintenance.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS library_instance_maintenance; \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000072_library_instance_maintenance.up.sql b/internal/platform/postgres/migrations/000072_library_instance_maintenance.up.sql new file mode 100644 index 0000000..f1357b5 --- /dev/null +++ b/internal/platform/postgres/migrations/000072_library_instance_maintenance.up.sql @@ -0,0 +1,21 @@ +-- 维护占用必须按"实例"互斥,而不是按库 scope。 +-- 旧表的 scope 主键允许 internal/external 各持一个窗口,于是两个作业会同时 +-- 停/起同一个实例的写入者——那是最危险的一种假互斥。 +CREATE TABLE library_instance_maintenance ( + instance VARCHAR(128) PRIMARY KEY, + owner_job_id VARCHAR(128) NOT NULL, + owner_scope VARCHAR(16) NOT NULL CHECK (owner_scope IN ('internal','external')), + target_fingerprint VARCHAR(64) NOT NULL, + level VARCHAR(16) NOT NULL DEFAULT 'instance' CHECK (level IN ('library','instance')), + services JSONB NOT NULL DEFAULT '[]'::jsonb, + ingress_state VARCHAR(16) NOT NULL DEFAULT 'unknown' CHECK (ingress_state IN ('unknown','closed','open')), + state VARCHAR(24) NOT NULL CHECK (state IN ('acquired','applying','verified','released','needs_restore','recovering')), + original_service_state TEXT NOT NULL DEFAULT '', + note TEXT NOT NULL DEFAULT '', + acquired_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + released_at TIMESTAMPTZ +); + +-- 把已有窗口按 scope 行回填成实例行不可行(历史 target_identity 是自由文本), +-- 这里只保证新窗口从第一刻起就按实例互斥;旧行保留为审计记录。 \ No newline at end of file diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index 1aceceb..6e64155 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,30 +11,35 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 71 { - t.Fatalf("migration count = %d, want 71", len(migrations)) + if len(migrations) != 72 { + t.Fatalf("migration count = %d, want 72", len(migrations)) + } + // 72 让维护占用按实例互斥(internal/external 不再各有各的锁)。 + if instance := migrations[len(migrations)-1]; instance.Version != 72 || + !strings.Contains(instance.UpSQL, "library_instance_maintenance") { + t.Fatalf("migration 72 = %#v", instance) } // 71 让维护记录能区分实例级维护与仅治理库互斥。 - if level := migrations[len(migrations)-1]; level.Version != 71 || + if level := migrations[len(migrations)-2]; level.Version != 71 || !strings.Contains(level.UpSQL, "level") || !strings.Contains(level.UpSQL, "ingress_state") { t.Fatalf("migration 71 = %#v", level) } - if maint := migrations[len(migrations)-2]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") { + if maint := migrations[len(migrations)-3]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") { t.Fatalf("migration 70 = %#v", maint) } - if tail := migrations[len(migrations)-3]; tail.Version != 69 || + if tail := migrations[len(migrations)-4]; tail.Version != 69 || !strings.Contains(tail.UpSQL, "library_jobs_write_target") { t.Fatalf("migration 69 = %#v", tail) } - if restore := migrations[len(migrations)-4]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { + if restore := migrations[len(migrations)-5]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { t.Fatalf("migration 68 = %#v", restore) } - if last := migrations[len(migrations)-5]; last.Version != 67 || + if last := migrations[len(migrations)-6]; last.Version != 67 || !strings.Contains(last.UpSQL, "library_release_authorizations") || !strings.Contains(last.UpSQL, "params_digest") { t.Fatalf("migration 67 = %#v", last) } - if flow := migrations[len(migrations)-6]; flow.Version != 66 || + if flow := migrations[len(migrations)-7]; flow.Version != 66 || !strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") || !strings.Contains(flow.UpSQL, "library_jobs_active") { t.Fatalf("migration 66 = %#v", flow)