diff --git a/CHANGELOG.md b/CHANGELOG.md index 301332c..bf775f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +## 开发快照(目标维护:实例级窗口) + +### Added + +- 同步/恢复不再只锁治理库 scope:`runLibraryRelease` 现在先开**实例级维护窗口** + (`internal/librarymaintenance` + `internal/libraryfreeze` 复用)—— + 持久化 owner / 目标精确身份(实例、写入者容器、入口、库/索引/桶 + 指纹)/ 原服务状态, + 关闭目标入口(SIGTERM 后监听立即关闭)、停下全部声明写入者, + 并验证「写入者真的停了」「在途请求真的排空了」「入口真的连不上」之后才允许写目标。 +- `libraryfreeze.RunWindow`:一个窗口里跑调用方的写阶段。第一次写目标前必须 + `MarkContentRisk`(fsync 落盘);写完先恢复原运行状态与入口,再释放维护占用。 +- 入口探针(`internal/librarymaintenance`):关闭 = 连不上(代理还回 502 不算关闭), + 恢复 = HTTP 200;只有原本在跑的服务会被重启。 +- 迁移 `000071_library_maintenance_level`:维护记录新增 `level`(instance/library)、 + `target_fingerprint`、`services`、`ingress_state`,页面/审计能看清这次到底动的是实例还是只锁了一个库。 + +### Fixed + +- 进程被杀/授权过期不再误放行半写目标:journal 里的 `content_risk` 一旦置位, + 重启恢复只收尾确定性的事,**不会**自动把写入者启动回来;原作业接管后才继续。 +- 回滚失败(`rollback_failed`)或校验失败(`finalize` 不一致)时,服务保持停止、 + 维护占用保持持有(`needs_restore`),入口不重新开放。 +- 接管场景(已有预备份)一律按「内容可能半写」处理,任何早退都不重新开放目标。 + +### 限制(本轮不承诺) + +- 维护层级判定是「配置里声明的写入者组」:多实例/多写入者拓扑变更需要重新声明, + 不会自动猜测新容器。 +- 前端只透传新增字段,页面尚未为 `level/ingress_state` 做专门的文案与布局; + Wiki 语义重算与浏览器整体验收仍未完成。 +- 本机专用候选实例(kgmaint-*)仅承载验收数据,不代表远程生产已验证。 + ## 开发快照(读接口超时:真实修复) ### Fixed diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index 20e918f..b32f6fc 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -12,11 +12,49 @@ import ( "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" + "knowledge-governance/internal/libraryfreeze" + "knowledge-governance/internal/librarymaintenance" "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarysnapshot" postgresstore "knowledge-governance/internal/platform/postgres" ) +// 目标内容在窗口内的几种"已知状态",决定目标能否被重新开放: +// - contentUntouched:这次窗口还没写过目标(可以安全开放) +// - contentSucceeded / contentRolledBack:内容已验证,或已回滚并验证(可以开放) +// - contentHalfWritten / contentRollbackFails:内容可能半写(必须保持维护) +const ( + contentUntouched = "" + contentSucceeded = "succeeded" + contentRolledBack = "rolled_back" + contentHalfWritten = "half_written" + contentRollbackFails = "rollback_failed" +) + +// contentTrackedTarget 只做一件事:在第一次写目标时把"内容有风险"标出来。 +// 这样"授权过期/进程被杀/校验失败"这些早退路径不会把半写目标当成没动过。 +type contentTrackedTarget struct { + inner libraryrelease.Target + onApply func() +} + +func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) { + return t.inner.Current(ctx) +} + +func (t *contentTrackedTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error { + t.onApply() + return t.inner.Apply(ctx, stage, snapshotID) +} + +func (t *contentTrackedTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error { + return t.inner.Verify(ctx, expect) +} + +func (t *contentTrackedTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error { + return t.inner.Restore(ctx, backup) +} + // libraryReleasePayload 是同步与恢复共用的队列载荷形状。 type libraryReleasePayload struct { LibraryJobID string `json:"library_job_id"` @@ -27,7 +65,28 @@ type libraryReleasePayload struct { FinalizeOnly bool `json:"finalize_only,omitempty"` } -// runLibraryRelease 执行"闸口 → 授权 → 预备份 → 四类写入 → 校验"的完整链路。 +// openLibraryMaintenance 按部署注入的配置装配实例级维护目标。 +// 未接线时返回明确原因:只锁治理库 scope 不算维护,宁可拒绝写也不冒充。 +func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) { + database, err := librarymaintenance.DatabaseIdentity(config.LibraryProductionDatabaseURL) + if err != nil { + return nil, err + } + return librarymaintenance.New(librarymaintenance.Config{ + Instance: config.LibraryTargetInstance, + Project: config.LibraryTargetProject, + Writers: config.LibraryTargetWriters, + NonWriters: config.LibraryTargetNonWriters, + DockerSocket: config.LibraryTargetDockerSocket, + IngressURL: config.LibraryTargetIngressURL, + Database: database, + Index: config.LibraryProductionIndex, + Bucket: config.LibraryProductionS3Bucket, + }) +} + +// runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 → +// 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。 // 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。 func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job, kind governance.LibraryJobKind) error { @@ -106,134 +165,238 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a } target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping) if wireErr == nil { - if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { - recorder.mappingDigest = digest + if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { + recorder.mappingDigest = mapping } } if wireErr != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup") } defer target.Close() - // 维护窗口:目标在同步/恢复期间被本作业独占,任何其他写入一律拒绝。 + + // 维护目标必须在真正动目标之前确定:缺配置就拒绝写,不用"库级锁"顶替。 + manager, managerErr := openLibraryMaintenance(config) + if managerErr != nil { + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, + "目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup") + } targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope, redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket) + originalState, stateErr := manager.OriginalState(ctx) + if stateErr != nil { + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, + "读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup") + } + identity := manager.Target() maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{ - Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity, - State: "acquired", OriginalServiceState: "candidate: no external writer configured", - Note: "同步/恢复期间目标维护,验证通过才开放", + Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, + TargetIdentity: targetIdentity + " | " + manager.Description(), + Level: librarymaintenance.LevelInstance, + TargetFingerprint: identity.Fingerprint, + Services: identity.Writers, + IngressState: "unknown", + State: "acquired", + OriginalServiceState: originalState, + Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放", }, now) if acquireErr != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup") } _ = maintenance - if payload.FinalizeOnly { - // 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。 - recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致") - if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil { - message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error() - recorder.Stage(ctx, "finalize", "failed", message) - _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, - "生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) - return errors.New(message) + coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot) + if openErr != nil { + // 协调器都打不开:这一次没动过目标。接管场景除外——上一次尝试可能已经把 + // 写入者停住并写了一半,那种情况下保持占用是唯一安全的选择。 + if resumed { + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", + "维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC()) + } else { + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified", + "未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC()) + _ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC()) } - mappingDigest, _ := target.Search().MappingSHA256(ctx) - if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ - ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), - Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, - SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker", - Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only", - "index_mapping_sha256": mappingDigest}, - CreatedAt: time.Now().UTC(), - }); err != nil { - message := "收尾续作仍无法登记生效版本:" + err.Error() - recorder.Stage(ctx, "finalize", "failed", message) - _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, - "预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) - return errors.New(message) - } - recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)") - _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", - previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) - // 只有校验+登记都完成才开放目标。 - _ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC()) - return nil + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "维护协调器不可用:"+openErr.Error(), "prebackup") } + defer control.Close() + defer coordinator.Close() - var backupSource libraryrelease.BackupStore = backups + // 接管场景下目标可能已经被上一次尝试写过:在证明"已回滚或已验证"之前, + // 一律按半写对待,任何早退都不许重新开放目标。 + contentState := contentUntouched if resumed { - backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID, - SnapshotID: previous.SnapshotID}} + contentState = contentHalfWritten + } + tracked := &contentTrackedTarget{inner: target, onApply: func() { contentState = contentHalfWritten }} + allowRestore := func(error) bool { + return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack } - request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID, - ParamsDigest: digest, Now: now, - Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID, - ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}} - _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now) - outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder) - finishedAt := time.Now().UTC() - switch outcome.State { - case "succeeded": - mappingDigest := "" - if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { - mappingDigest = digest - } - // 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。 - // 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。 - recorder.Stage(ctx, "finalize", "running", "登记生产生效版本") - versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ - ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), - Scope: payload.Scope, - Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, - SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy, - Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID, - "backup_bytes": outcome.Backup.Bytes, "action": string(kind), - "index_mapping_sha256": mappingDigest}, - CreatedAt: finishedAt, - }) - if versionErr != nil { - message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error() - recorder.Stage(ctx, "finalize", "failed", message) - // 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。 - if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", - message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记", - outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { - log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr) + execute := func(ctx context.Context) error { + if payload.FinalizeOnly { + // 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。 + recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致") + if verifyErr := tracked.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil { + contentState = contentHalfWritten + message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error() + recorder.Stage(ctx, "finalize", "failed", message) + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, + "生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) + return errors.New(message) } - return errors.New(message) + contentState = contentSucceeded + mappingDigest, _ := target.Search().MappingSHA256(ctx) + if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ + ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), + Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, + SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker", + Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only", + "index_mapping_sha256": mappingDigest}, + CreatedAt: time.Now().UTC(), + }); err != nil { + message := "收尾续作仍无法登记生效版本:" + err.Error() + recorder.Stage(ctx, "finalize", "failed", message) + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, + "预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) + return errors.New(message) + } + recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)") + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", + previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) + return nil } - recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本") - if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", - outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { - // 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。 - return fmt.Errorf("作业状态写入失败: %w", advanceErr) + + var backupSource libraryrelease.BackupStore = backups + if resumed { + backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID, + SnapshotID: previous.SnapshotID}} } - // 验证通过 + 版本登记成功 = 唯一开放目标的路径。 - _ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, finishedAt) - log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind, - payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID) - return nil - case "rolled_back": - _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", - outcome.Error, outcome.Rollback, "", "", finishedAt) - _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", - "已回滚到上一版本,但验证未通过:目标继续维护,人工确认后再开放", finishedAt) - return errors.New(outcome.Error) - case "rollback_failed": - _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", - outcome.Error, outcome.Rollback, "", "", finishedAt) - _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", - "回滚失败:目标保持维护,必须按预备份恢复", finishedAt) - return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback) + + request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID, + ParamsDigest: digest, Now: now, + Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID, + ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}} + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now) + outcome := libraryrelease.Run(ctx, request, tracked, backupSource, recorder) + finishedAt := time.Now().UTC() + switch outcome.State { + case "succeeded": + contentState = contentSucceeded + mappingDigest := "" + if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { + mappingDigest = mapping + } + // 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。 + // 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。 + recorder.Stage(ctx, "finalize", "running", "登记生产生效版本") + versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ + ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), + Scope: payload.Scope, + Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, + SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy, + Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID, + "backup_bytes": outcome.Backup.Bytes, "action": string(kind), + "index_mapping_sha256": mappingDigest}, + CreatedAt: finishedAt, + }) + if versionErr != nil { + message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error() + recorder.Stage(ctx, "finalize", "failed", message) + // 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。 + if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", + message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记", + outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { + log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr) + } + return errors.New(message) + } + recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本") + if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", + outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { + // 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。 + return fmt.Errorf("作业状态写入失败: %w", advanceErr) + } + log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind, + payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID) + return nil + case "rolled_back": + contentState = contentRolledBack + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", + outcome.Error, outcome.Rollback, "", "", finishedAt) + return errors.New(outcome.Error) + case "rollback_failed": + contentState = contentRollbackFails + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", + outcome.Error, outcome.Rollback, "", "", finishedAt) + return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback) + default: + recovery := outcome.Recovery + if recovery == "" { + recovery = "未对生产执行任何写入,无需恢复" + } + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", + outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt) + return errors.New(outcome.Error) + } + } + + prepare := func(ctx context.Context) error { + // 停下写入者之前必须确认目标现在是健康的:拿一个已经不健康的目标开窗口, + // 只会把"本来坏的"记成"维护造成的"。 + return manager.VerifyRunning(ctx) + } + windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) { + if err := w.MarkContentRisk(ctx); err != nil { + return "", err + } + if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed", + "目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil { + // 记录失败不影响隔离本身,但必须留痕,不能假装记上了。 + log.Printf("library %s maintenance ingress record failed: %v", payload.LibraryJobID, ingressErr) + } + return "", execute(ctx) + } + window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore) + + // 收尾:只有"内容已验证/已回滚"且服务与入口都恢复,才释放维护占用。 + now2 := time.Now().UTC() + switch { + case window.State == "held": + note := "目标保持维护:内容状态未验证(半写或回滚失败),或服务/入口未能恢复" + if windowErr != nil { + note += ":" + windowErr.Error() + } + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2) + if windowErr == nil { + return errors.New(note) + } + return windowErr + case contentState == contentSucceeded: + _ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "目标入口已恢复并验证", now2) + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified", + "写入与校验完成,目标已按原运行状态恢复", now2) + if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil { + return fmt.Errorf("目标已恢复但维护占用释放失败: %w", err) + } + return windowErr + case contentState == contentRolledBack: + _ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "已回滚并验证,目标入口恢复", now2) + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified", + "本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2) + if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil { + return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err) + } + return windowErr default: - recovery := outcome.Recovery - if recovery == "" { - recovery = "未对生产执行任何写入,无需恢复" + // 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。 + _ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", + "未进入维护窗口,目标未被停用", now2) + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified", + "未写目标,维护窗口未开启", now2) + if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil { + return fmt.Errorf("维护占用释放失败: %w", err) } - _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", - outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt) - return errors.New(outcome.Error) + return windowErr } } diff --git a/internal/appconfig/config.go b/internal/appconfig/config.go index f7c6806..7d46ae7 100644 --- a/internal/appconfig/config.go +++ b/internal/appconfig/config.go @@ -94,6 +94,15 @@ type Config struct { // LibraryRetentionVersions / Days 是保留策略;都为 0 表示未设置(不生成清理候选)。 LibraryRetentionVersions int LibraryRetentionDays int + // 目标维护(实例级):同步/恢复期间关闭目标入口、停下写入者、排空在途请求。 + // 任一项留空即"未接线",生产写入会被拒绝——只锁治理库 scope 不算维护。 + LibraryTargetInstance string + LibraryTargetProject string + LibraryTargetWriters []string + LibraryTargetNonWriters map[string]string + LibraryTargetDockerSocket string + LibraryTargetIngressURL string + LibraryTargetJournalRoot string // LibraryAgentSocket 是主机侧封存 agent 的 unix socket(只由 worker 连接)。 // 留空即"未接线":页面不显示可用的封存入口,接口返回原因而不是假装已排队。 LibraryAgentSocket string @@ -182,6 +191,13 @@ func Load(getenv func(string) string) (Config, error) { LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")), LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")), LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")), + LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")), + LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")), + LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")), + LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")), + LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"), + LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")), + LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")), LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"), LibraryEmbeddingDimension: embeddingDimension, LibraryCapacityBudgetBytes: capacityBudget, @@ -302,6 +318,39 @@ func Load(getenv func(string) string) (Config, error) { if config.LibraryAgentSocket != "" && !filepath.IsAbs(config.LibraryAgentSocket) { return Config{}, errors.New("LIBRARY_AGENT_SOCKET 必须是绝对路径(unix socket 由主机侧 agent 持有)") } + // 目标维护要么完整配置,要么完全不配;半配置会让窗口"看起来存在"却挡不住目标。 + targetSet := config.LibraryTargetInstance != "" || config.LibraryTargetProject != "" || + len(config.LibraryTargetWriters) > 0 || config.LibraryTargetIngressURL != "" + if targetSet { + missing := make([]string, 0, 4) + if config.LibraryTargetInstance == "" { + missing = append(missing, "LIBRARY_TARGET_INSTANCE") + } + if config.LibraryTargetProject == "" { + missing = append(missing, "LIBRARY_TARGET_PROJECT") + } + if len(config.LibraryTargetWriters) == 0 { + missing = append(missing, "LIBRARY_TARGET_WRITERS") + } + if config.LibraryTargetIngressURL == "" { + missing = append(missing, "LIBRARY_TARGET_INGRESS_URL") + } + if config.LibraryTargetJournalRoot == "" { + missing = append(missing, "LIBRARY_TARGET_JOURNAL_ROOT") + } + if len(missing) > 0 { + return Config{}, fmt.Errorf("目标维护配置不完整:缺少 %s", strings.Join(missing, "、")) + } + if !filepath.IsAbs(config.LibraryTargetJournalRoot) { + return Config{}, errors.New("LIBRARY_TARGET_JOURNAL_ROOT 必须是绝对路径") + } + if !filepath.IsAbs(config.LibraryTargetDockerSocket) { + return Config{}, errors.New("LIBRARY_TARGET_DOCKER_SOCKET 必须是绝对路径") + } + if len(config.LibraryTargetNonWriters) == 0 { + return Config{}, errors.New("LIBRARY_TARGET_NON_WRITERS 必须逐项说明项目内非写入服务") + } + } if config.TestWeKnoraHTTPTimeout, err = durationOrDefault(getenv("TEST_WEKNORA_HTTP_TIMEOUT"), config.TestWeKnoraHTTPTimeout); err != nil { return Config{}, fmt.Errorf("TEST_WEKNORA_HTTP_TIMEOUT: %w", err) } @@ -419,6 +468,33 @@ func splitOrigins(value string) []string { return origins } +// splitList 把逗号分隔的声明拆成去空列表(顺序保留,便于日志对账)。 +func splitList(value string) []string { + parts := strings.Split(value, ",") + out := make([]string, 0, len(parts)) + for _, part := range parts { + if trimmed := strings.TrimSpace(part); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} + +// splitDeclarations 解析 "name=reason,name2=reason2"。 +// 没有理由的声明直接丢空,由 Validate 拒绝——"忘记声明"不能变成"不用声明"。 +func splitDeclarations(value string) map[string]string { + out := map[string]string{} + for _, item := range splitList(value) { + name, reason, ok := strings.Cut(item, "=") + name, reason = strings.TrimSpace(name), strings.TrimSpace(reason) + if !ok || name == "" || reason == "" { + continue + } + out[name] = reason + } + return out +} + // valueOrDefaultInt 解析正整数(非法/缺省都退回默认值)。 func valueOrDefaultInt(value string, fallback int) int { parsed, err := strconv.Atoi(strings.TrimSpace(value)) diff --git a/internal/libraryfreeze/docker.go b/internal/libraryfreeze/docker.go index 3a293ff..be18ca1 100644 --- a/internal/libraryfreeze/docker.go +++ b/internal/libraryfreeze/docker.go @@ -78,6 +78,7 @@ type dockerInspection struct { Restarting bool Dead bool Status string + ExitCode int Health *struct{ Status string } } `json:"State"` } @@ -104,7 +105,14 @@ func (d *DockerController) inspect(ctx context.Context, id string) (Service, err return Service{}, errors.New("contradictory test writer state") } healthy := data.State.Running && data.State.Health != nil && data.State.Health.Status == "healthy" - return Service{ID: id, Running: data.State.Running, Healthy: healthy}, nil + // 退出码只在"已经停下"时才有意义:它证明服务是自己收尾退出的(0), + // 还是被信号切断的(143/137)。运行中的服务不提供这项证据。 + var exit *int + if !data.State.Running { + code := data.State.ExitCode + exit = &code + } + return Service{ID: id, Running: data.State.Running, Healthy: healthy, ExitCode: exit}, nil } func (d *DockerController) Inventory(ctx context.Context) ([]Service, error) { filters, _ := json.Marshal(map[string][]string{"label": {sourceLabel + "=" + d.source}}) diff --git a/internal/libraryfreeze/freeze.go b/internal/libraryfreeze/freeze.go index 9d6a17d..27a9df1 100644 --- a/internal/libraryfreeze/freeze.go +++ b/internal/libraryfreeze/freeze.go @@ -9,7 +9,9 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "io" + "net/url" "os" "path/filepath" "sort" @@ -22,6 +24,9 @@ type Service struct { ID string `json:"id"` Running bool `json:"running"` Healthy bool `json:"healthy"` + // ExitCode 是"停下之后"的进程退出码,只有控制器真的读得到时才填。 + // 它是在途请求是否被排空的证据:0 = 服务自己收尾退出,143/137 = 被切断。 + ExitCode *int `json:"exit_code,omitempty"` } // Inventory must enumerate the complete, independently scoped test writer set, @@ -47,15 +52,29 @@ type journal struct { Source string `json:"source"` Job string `json:"job"` Result - Services []serviceRecord `json:"services"` - StartedAt time.Time `json:"started_at"` - FinishedAt *time.Time `json:"finished_at,omitempty"` - Code string `json:"code,omitempty"` + // Level 区分"实例级维护"(写入者/入口被真的关掉)与"仅治理库互斥"。 + // 没有 Level 的旧 journal 来自封存链路,按实例级处理但不冒充已验证。 + Level string `json:"level,omitempty"` + Ingress string `json:"ingress,omitempty"` + IngressClosed bool `json:"ingress_closed,omitempty"` + DrainVerified bool `json:"drain_verified,omitempty"` + ContentRisk bool `json:"content_risk,omitempty"` + RestoreBlocked bool `json:"restore_blocked,omitempty"` + Services []serviceRecord `json:"services"` + StartedAt time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Code string `json:"code,omitempty"` } type Coordinator struct { root *os.Root source string control Controller + level string + ingress string + // ingressClosed / ingressReady 是可选的外部探针:声明了实例级维护就必须 + // 证明入口真的关了、真的回来了,而不是"停了容器就算关"。 + ingressClosed func(context.Context) error + ingressReady func(context.Context) error } func Open(root, source string, c Controller) (*Coordinator, error) { @@ -78,6 +97,30 @@ func Open(root, source string, c Controller) (*Coordinator, error) { } func (c *Coordinator) Close() error { return c.root.Close() } +// SetTarget 声明这次维护的层级与入口。实例级维护必须指名入口地址; +// 只锁治理 scope 的维护不许带入口,否则就是拿"库级"冒充"实例级"。 +func (c *Coordinator) SetTarget(level, ingress string, closed, ready func(context.Context) error) error { + switch level { + case "instance": + if !validIngress(ingress) { + return errors.New("instance maintenance requires the target ingress address") + } + case "library": + if ingress != "" || closed != nil || ready != nil { + return errors.New("library scope lock must not claim ingress control") + } + default: + return errors.New("invalid maintenance level") + } + c.level, c.ingress, c.ingressClosed, c.ingressReady = level, ingress, closed, ready + return nil +} + +func validIngress(raw string) bool { + parsed, err := url.Parse(raw) + return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.Path != "" && parsed.User == nil +} + func (c *Coordinator) Recover(ctx context.Context) error { lock, err := c.lock() if err != nil { @@ -112,6 +155,13 @@ func (c *Coordinator) Recover(ctx context.Context) error { pending = append(pending, j) } } + // 内容可能已被写过的窗口不能自动开放:重启恢复只许"收尾确定性的事", + // 半写目标必须由原作业接管或人工按预备份恢复,否则就是把半写当成品放行。 + for _, j := range pending { + if j.ContentRisk || j.RestoreBlocked { + return fmt.Errorf("维护窗口 %s 的内容状态未验证(可能已半写):目标保持停用,须由原作业接管或按预备份恢复", j.Job) + } + } if len(pending) > 1 { return errors.New("multiple interrupted freezes require operator reconciliation") } @@ -245,10 +295,23 @@ func (c *Coordinator) load(job string) (journal, error) { return journal{}, errors.New("freeze journal identity mismatch") } switch j.State { - case "stopping", "capturing", "restoring", "succeeded", "failed": + case "stopping", "held", "capturing", "restoring", "succeeded", "failed": default: return journal{}, errors.New("invalid freeze journal state") } + if j.Level != "" && j.Level != "instance" && j.Level != "library" { + return journal{}, errors.New("invalid freeze maintenance level") + } + // 声明了实例级维护就必须留下入口证据;只锁库的窗口不许带入口。 + if j.Level == "instance" && !validIngress(j.Ingress) { + return journal{}, errors.New("invalid freeze ingress evidence") + } + if j.Level == "library" && (j.Ingress != "" || j.IngressClosed) { + return journal{}, errors.New("library scope lock claims ingress control") + } + if j.RestoreBlocked && !j.ContentRisk { + return journal{}, errors.New("invalid freeze restore block") + } if len(j.Services) == 0 || len(j.Services) > 256 { return journal{}, errors.New("invalid freeze journal services") } @@ -256,6 +319,10 @@ func (c *Coordinator) load(job string) (journal, error) { if !validID(s.ID) || (i > 0 && j.Services[i-1].ID >= s.ID) || s.StopRequested && !s.Running || s.Restored && (!s.StopRequested || !s.Running) { return journal{}, errors.New("invalid freeze service ownership") } + // 退出码只能出现在"被要求停过"的服务上;Running 记的是原始状态。 + if s.ExitCode != nil && !s.StopRequested { + return journal{}, errors.New("invalid freeze drain evidence") + } } return j, nil } @@ -379,113 +446,3 @@ func (c *Coordinator) restore(ctx context.Context, j *journal) error { } return failures } -func (c *Coordinator) Run(ctx context.Context, job string, capture func(context.Context) (string, error)) (Result, error) { - return c.RunPrepared(ctx, job, func(context.Context) error { return nil }, capture) -} - -// RunPrepared runs readiness checks under the source lock, before any stop -// intent. It does not repeat checks or capture for an already completed job. -func (c *Coordinator) RunPrepared(ctx context.Context, job string, prepare func(context.Context) error, capture func(context.Context) (string, error)) (Result, error) { - if !validID(job) || capture == nil || prepare == nil { - return Result{}, errors.New("invalid freeze request") - } - if err := ctx.Err(); err != nil { - return Result{}, err - } - lock, err := c.lock() - if err != nil { - return Result{}, err - } - defer unlock(lock) - previous, err := c.load(job) - if err == nil { - if previous.State == "succeeded" { - return previous.Result, nil - } - return previous.Result, errors.New("freeze request already exists; recovery or a new request is required") - } - if !errors.Is(err, os.ErrNotExist) { - return Result{}, err - } - if err = c.pending(); err != nil { - return Result{}, err - } - if err = prepare(ctx); err != nil { - return Result{}, err - } - if err = ctx.Err(); err != nil { - return Result{}, err - } - items, err := inventory(ctx, c.control) - if err != nil { - return Result{}, err - } - j := journal{Version: 1, Source: c.source, Job: job, Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()} - for _, s := range items { - if s.Running && !s.Healthy { - return Result{}, errors.New("test writer is not ready") - } - j.Services = append(j.Services, serviceRecord{Service: s}) - } - if err = c.save(j); err != nil { - return Result{}, err - } - var workErr error - for i := range j.Services { - if !j.Services[i].Running { - continue - } - j.Services[i].StopRequested = true - if workErr = c.save(j); workErr != nil { - break - } - if workErr = c.control.Stop(ctx, j.Services[i].ID); workErr != nil { - break - } - } - if workErr == nil { - workErr = c.stopped(ctx, j) - } - if workErr == nil { - for i := range j.Services { - if j.Services[i].StopRequested { - j.Services[i].StopVerified = true - } - } - j.State = "capturing" - workErr = c.save(j) - } - if workErr == nil { - j.SnapshotID, workErr = invokeCapture(ctx, capture) - } - if workErr == nil { - workErr = ctx.Err() - } - if workErr == nil { - decoded, decodeErr := hex.DecodeString(j.SnapshotID) - if decodeErr != nil || len(decoded) != 32 || strings.ToLower(j.SnapshotID) != j.SnapshotID { - workErr = errors.New("invalid candidate snapshot digest") - } - } - if workErr == nil { - workErr = c.stopped(ctx, j) - } - recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) - defer cancel() - recoveryErr := c.restore(recoveryCtx, &j) - if recoveryErr != nil { - return j.Result, errors.Join(workErr, recoveryErr) - } - if workErr != nil { - j.State = "failed" - j.Code = "capture_or_isolation_failed" - } else { - j.State = "succeeded" - } - ended := time.Now().UTC() - j.FinishedAt = &ended - if err = c.save(j); err != nil { - return j.Result, err - } - return j.Result, workErr -} diff --git a/internal/libraryfreeze/project.go b/internal/libraryfreeze/project.go index 678576f..4507e23 100644 --- a/internal/libraryfreeze/project.go +++ b/internal/libraryfreeze/project.go @@ -113,6 +113,7 @@ type projectInspection struct { Restarting bool Dead bool Status string + ExitCode int Health *struct{ Status string } } `json:"State"` } @@ -158,7 +159,14 @@ func (p *ProjectController) inspect(ctx context.Context, id string) (Service, er return Service{}, errors.New("contradictory project writer state") } healthy := data.State.Running && data.State.Health != nil && data.State.Health.Status == "healthy" - return Service{ID: id, Running: data.State.Running, Healthy: healthy}, nil + // 退出码只在"已经停下"时才有意义:它证明服务是自己收尾退出的(0), + // 还是被信号切断的(143/137)。运行中的服务不提供这项证据。 + var exit *int + if !data.State.Running { + code := data.State.ExitCode + exit = &code + } + return Service{ID: id, Running: data.State.Running, Healthy: healthy, ExitCode: exit}, nil } // Inventory enumerates the complete project writer group and rejects an diff --git a/internal/libraryfreeze/window.go b/internal/libraryfreeze/window.go new file mode 100644 index 0000000..620b492 --- /dev/null +++ b/internal/libraryfreeze/window.go @@ -0,0 +1,324 @@ +package libraryfreeze + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "os" + "strings" + "time" +) + +// Window 是一次维护窗口里"写目标"阶段的句柄。 +// +// work 必须在第一笔目标写入之前调用 MarkContentRisk:它是一条持久证据, +// 记录"从这里开始目标内容可能已被改动"。进程在这之后死掉时,恢复器不会 +// 自动把目标开放——半写目标只能由原作业接管或按预备份恢复。 +type Window struct { + c *Coordinator + j *journal +} + +func (w *Window) MarkContentRisk(ctx context.Context) error { + if w == nil || w.c == nil || w.j == nil { + return errors.New("invalid maintenance window") + } + if err := ctx.Err(); err != nil { + return err + } + if w.j.ContentRisk { + return nil + } + w.j.ContentRisk = true + return w.c.save(*w.j) +} + +// Job 返回窗口所属作业 id(恢复器与调用方据此对账)。 +func (w *Window) Job() string { + if w == nil || w.j == nil { + return "" + } + return w.j.Job +} + +func (c *Coordinator) Run(ctx context.Context, job string, capture func(context.Context) (string, error)) (Result, error) { + return c.RunPrepared(ctx, job, func(context.Context) error { return nil }, capture) +} + +// RunPrepared 是封存链路的入口:停止写入者 → 在窗口内取快照 → 恢复写入者。 +// 它等价于 allowRestore 恒为 true 的 RunWindow。 +func (c *Coordinator) RunPrepared(ctx context.Context, job string, prepare func(context.Context) error, capture func(context.Context) (string, error)) (Result, error) { + if capture == nil { + return Result{}, errors.New("invalid freeze request") + } + return c.RunWindow(ctx, job, prepare, func(ctx context.Context, _ *Window) (string, error) { + return capture(ctx) + }, func(error) bool { return true }) +} + +// RunWindow 是维护窗口的唯一实现: +// +// 关入口 + 停全部写入者 → 验证真的停了、在途真的排空了 → 跑 work +// → 恢复"原本在跑"的服务 → 验证入口回来 +// +// allowRestore 在 work 失败后决定目标能否重新开放:返回 false 表示内容状态 +// 未验证(半写)——此时服务保持停止、journal 保持非终态,崩溃恢复也不许 +// 自动启动它们。同一个 job 可以接管未完成的窗口,别的 job 不行。 +func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(context.Context) error, + work func(context.Context, *Window) (string, error), allowRestore func(error) bool) (Result, error) { + if !validID(job) || work == nil || prepare == nil { + return Result{}, errors.New("invalid freeze request") + } + if allowRestore == nil { + allowRestore = func(error) bool { return true } + } + if err := ctx.Err(); err != nil { + return Result{}, err + } + lock, err := c.lock() + if err != nil { + return Result{}, err + } + defer unlock(lock) + + previous, loadErr := c.load(job) + j := journal{} + switch { + case loadErr == nil && previous.State == "succeeded": + return previous.Result, nil + case loadErr == nil && (previous.State == "stopping" || previous.State == "held"): + // 接管:目标必须仍然停住。不重新"准备"、也不从零重来。 + j = previous + if j.State == "held" && j.RestoreBlocked { + // 上一次窗口把内容判成半写:接管方必须自己重新证明,不许清掉风险。 + j.RestoreBlocked = false + } + case loadErr == nil: + return previous.Result, errors.New("freeze request already exists; recovery or a new request is required") + case !errors.Is(loadErr, os.ErrNotExist): + return Result{}, loadErr + default: + if err = c.pending(); err != nil { + return Result{}, err + } + if err = prepare(ctx); err != nil { + return Result{}, err + } + if err = ctx.Err(); err != nil { + return Result{}, err + } + items, err := inventory(ctx, c.control) + if err != nil { + return Result{}, err + } + j = journal{Version: 1, Source: c.source, Job: job, Level: c.level, Ingress: c.ingress, + Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()} + for _, s := range items { + if s.Running && !s.Healthy { + return Result{}, errors.New("test writer is not ready") + } + j.Services = append(j.Services, serviceRecord{Service: s}) + } + if err = c.save(j); err != nil { + return Result{}, err + } + } + + // 停写入者 + 排空 + 关入口:任何一步不成立都不许往下写。 + if stopErr := c.stopAll(ctx, &j); stopErr != nil { + return c.abandonWindow(ctx, &j, stopErr, "stop_failed") + } + if stopErr := c.stopped(ctx, j); stopErr != nil { + return c.abandonWindow(ctx, &j, stopErr, "isolation_unverified") + } + if stopErr := c.verifyDrain(ctx, &j); stopErr != nil { + return c.abandonWindow(ctx, &j, stopErr, "drain_unverified") + } + if c.ingressClosed != nil { + if stopErr := c.ingressClosed(ctx); stopErr != nil { + return c.abandonWindow(ctx, &j, fmt.Errorf("目标入口仍然可用:%w", stopErr), "ingress_open") + } + j.IngressClosed = true + } + j.State = "held" + if err = c.save(j); err != nil { + return Result{}, err + } + + w := &Window{c: c, j: &j} + // work 里 panic 等同于失败:目标可能已被改过,必须走恢复分支,不能让 + // 进程直接崩掉把写入者留在停止状态。 + snapshotID, workErr := func() (id string, err error) { + defer func() { + if recover() != nil { + id, err = "", errors.New("维护窗口内的作业 panic") + } + }() + return work(ctx, w) + }() + if workErr != nil { + return c.closeWindow(ctx, &j, workErr, allowRestore(workErr), "") + } + if err = ctx.Err(); err != nil { + // 取消/超时发生在窗口内:内容状态未知,但写入者必须恢复(旧行为一致), + // 由上层把作业判失败;不把"没验证完"说成成功。 + return c.closeWindow(ctx, &j, err, true, "") + } + if snapshotID != "" { + if !validDigest(snapshotID) { + return c.closeWindow(ctx, &j, errors.New("invalid candidate snapshot digest"), true, "") + } + } + // work 报成功但写入者已经不在停住状态 = 隔离被破坏,不能登记成功。 + if err = c.stopped(ctx, j); err != nil { + return c.closeWindow(ctx, &j, err, true, "") + } + return c.closeWindow(ctx, &j, nil, true, snapshotID) +} + +// abandonWindow:在"还没写目标"的阶段失败——内容没被改过,可以安全恢复并结案。 +func (c *Coordinator) abandonWindow(ctx context.Context, j *journal, cause error, code string) (Result, error) { + recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + if restoreErr := c.restore(recoveryCtx, j); restoreErr != nil { + // 恢复不了就保持非终态:下一次恢复/接管还要处理它。 + j.Code = code + _ = c.save(*j) + return j.Result, errors.Join(cause, restoreErr) + } + ended := time.Now().UTC() + j.State = "failed" + j.Code = code + j.FinishedAt = &ended + if err := c.save(*j); err != nil { + return j.Result, errors.Join(cause, err) + } + return j.Result, cause +} + +// closeWindow:写完之后的收尾。allowRestore=false(半写且回滚失败)时 +// 服务保持停止、journal 保持非终态,返回的是"目标仍在维护"这个结论。 +func (c *Coordinator) closeWindow(ctx context.Context, j *journal, cause error, allowRestore bool, snapshotID string) (Result, error) { + if snapshotID != "" { + j.SnapshotID = snapshotID + } + if !allowRestore { + j.State = "held" + j.RestoreBlocked = true + if cause == nil { + cause = errors.New("目标内容状态未验证") + } + j.Code = "restore_blocked" + if err := c.save(*j); err != nil { + return j.Result, errors.Join(cause, err) + } + return j.Result, cause + } + recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + if restoreErr := c.restore(recoveryCtx, j); restoreErr != nil { + // 服务没能恢复 = 入口没回来 = 目标仍在维护,不许报成功。 + j.State = "held" + j.Code = "restore_failed" + if saveErr := c.save(*j); saveErr != nil { + return j.Result, errors.Join(cause, restoreErr, saveErr) + } + return j.Result, errors.Join(cause, restoreErr) + } + if c.ingressReady != nil { + if readyErr := c.ingressReady(recoveryCtx); readyErr != nil { + j.State = "held" + j.Code = "ingress_unverified" + _ = c.save(*j) + return j.Result, errors.Join(cause, fmt.Errorf("目标入口没有恢复:%w", readyErr)) + } + } + ended := time.Now().UTC() + j.FinishedAt = &ended + if cause != nil { + j.State = "failed" + if j.Code == "" { + j.Code = "work_failed" + } + } else { + j.State = "succeeded" + j.Code = "" + } + if err := c.save(*j); err != nil { + return j.Result, errors.Join(cause, err) + } + return j.Result, cause +} + +// stopAll 只对"原本在跑"的服务发停止指令,每发一个都先落盘意图: +// 进程在停止过程中死掉时,磁盘上仍能看出"哪些服务被要求停过"。 +func (c *Coordinator) stopAll(ctx context.Context, j *journal) error { + var failures error + for i := range j.Services { + if !j.Services[i].Running || j.Services[i].StopRequested { + continue + } + j.Services[i].StopRequested = true + if err := c.save(*j); err != nil { + return err + } + if err := c.control.Stop(ctx, j.Services[i].ID); err != nil { + failures = errors.Join(failures, err) + break + } + } + return failures +} + +// verifyDrain 用控制器报告的退出码判断"在途请求是否排空": +// 0 = 服务自己收尾退出;其它(143/137)= 被切断,不许当成排空。 +// 声明了实例级维护时,拿不到退出码就是没有证据——按未排空处理。 +func (c *Coordinator) verifyDrain(ctx context.Context, j *journal) error { + needed := 0 + for _, s := range j.Services { + if s.Running && s.StopRequested { + needed++ + } + } + if needed == 0 { + return nil + } + items, err := inventory(ctx, c.control) + if err != nil { + return err + } + actual := map[string]Service{} + for _, s := range items { + actual[s.ID] = s + } + verified := 0 + for i := range j.Services { + s := &j.Services[i] + if !s.Running || !s.StopRequested { + continue + } + now, ok := actual[s.ID] + if !ok || now.Running { + return errors.New("test writer still running") + } + if now.ExitCode == nil { + if c.level == "instance" { + return fmt.Errorf("无法证明 %s 的在途请求已排空(控制器没有退出码)", s.ID) + } + continue + } + s.ExitCode = now.ExitCode + if *now.ExitCode != 0 { + return fmt.Errorf("写入者 %s 未优雅退出(exit=%d):在途请求没有完成排空", s.ID, *now.ExitCode) + } + verified++ + } + j.DrainVerified = verified == needed + return nil +} + +func validDigest(id string) bool { + decoded, err := hex.DecodeString(id) + return err == nil && len(decoded) == 32 && strings.ToLower(id) == id +} diff --git a/internal/libraryfreeze/window_test.go b/internal/libraryfreeze/window_test.go new file mode 100644 index 0000000..6e7aebe --- /dev/null +++ b/internal/libraryfreeze/window_test.go @@ -0,0 +1,253 @@ +package libraryfreeze + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" +) + +// exitControl 模拟真实 Docker 控制器:停止时报告退出码(0 = 自己优雅退出)。 +type exitControl struct { + *fixtureControl + exit map[string]int +} + +func (c *exitControl) Inventory(ctx context.Context) ([]Service, error) { + items, err := c.fixtureControl.Inventory(ctx) + if err != nil { + return nil, err + } + for i := range items { + if code, ok := c.exit[items[i].ID]; ok && !items[i].Running { + value := code + items[i].ExitCode = &value + } + } + return items, nil +} + +func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) { + t.Helper() + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + c := &exitControl{fixtureControl: &fixtureControl{services: []Service{ + {ID: "app", Running: true, Healthy: true}, {ID: "store", Running: true, Healthy: true}}}, exit: map[string]int{"app": 0, "store": 0}} + engine, err := Open(root, "target-instance", c) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { engine.Close() }) + if err := engine.SetTarget("instance", "http://127.0.0.1:1/health", nil, nil); err != nil { + t.Fatal(err) + } + return root, c, engine +} + +// 维护窗口必须真的挡住写入:work 只能在所有写入者确认停下、入口确认关闭之后才开始。 +func TestWindowStopsWritersBeforeWorkAndRestoresAfter(t *testing.T) { + _, c, e := instanceFixture(t) + var sawStopped, sawClosed bool + e.ingressClosed = func(context.Context) error { sawClosed = true; return nil } + result, err := e.RunWindow(context.Background(), "job-1", func(context.Context) error { return nil }, + func(ctx context.Context, w *Window) (string, error) { + for _, s := range c.services { + if s.Running { + t.Fatal("work started while a writer was still running") + } + } + if err := w.MarkContentRisk(ctx); err != nil { + t.Fatal(err) + } + sawStopped = true + return strings.Repeat("b", 64), nil + }, func(error) bool { return true }) + if err != nil { + t.Fatal(err) + } + if !sawStopped || !sawClosed { + t.Fatalf("stopped=%v closed=%v", sawStopped, sawClosed) + } + if result.State != "succeeded" { + t.Fatalf("result=%+v", result) + } + j, err := e.load("job-1") + if err != nil { + t.Fatal(err) + } + if !j.DrainVerified || !j.IngressClosed || !j.ContentRisk || j.Level != "instance" { + t.Fatalf("journal evidence missing: %+v", j) + } + if strings.Join(c.calls, ",") != "stop:app,stop:store,start:app,start:store" { + t.Fatalf("calls=%v", c.calls) + } +} + +// 没有优雅退出证据(退出码非 0)时不许开始写目标:在途操作可能被切断。 +func TestWindowRefusesWorkWhenDrainUnverified(t *testing.T) { + _, c, e := instanceFixture(t) + c.exit["app"] = 143 + work := 0 + _, err := e.RunWindow(context.Background(), "job-drain", func(context.Context) error { return nil }, + func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true }) + if err == nil || !strings.Contains(err.Error(), "排空") { + t.Fatalf("err=%v", err) + } + if work != 0 { + t.Fatal("work ran without drain evidence") + } + for _, s := range c.services { + if !s.Running { + t.Fatal("writers were not restored after drain failure") + } + } +} + +// 入口停不下来(还在对外服务)就不是维护窗口。 +func TestWindowRefusesWorkWhenIngressStillServing(t *testing.T) { + _, c, e := instanceFixture(t) + e.ingressClosed = func(context.Context) error { return errors.New("ingress still serving") } + work := 0 + _, err := e.RunWindow(context.Background(), "job-ingress", func(context.Context) error { return nil }, + func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true }) + if err == nil || work != 0 { + t.Fatalf("err=%v work=%d", err, work) + } + for _, s := range c.services { + if !s.Running { + t.Fatal("writers were not restored after ingress failure") + } + } +} + +// 内容回滚失败:服务必须保持停止,且崩溃恢复不许自动把它们起来。 +func TestWindowKeepsWritersStoppedWhenRestoreBlocked(t *testing.T) { + root, c, e := instanceFixture(t) + _, err := e.RunWindow(context.Background(), "job-half", func(context.Context) error { return nil }, + func(ctx context.Context, w *Window) (string, error) { + if err := w.MarkContentRisk(ctx); err != nil { + t.Fatal(err) + } + return "", errors.New("目标写到一半,回滚也失败") + }, func(error) bool { return false }) + if err == nil { + t.Fatal("blocked restore reported success") + } + for _, s := range c.services { + if s.Running { + t.Fatalf("writer %s was restarted on a half-written target", s.ID) + } + } + j, err := e.load("job-half") + if err != nil || !j.RestoreBlocked || !j.ContentRisk { + t.Fatalf("journal=%+v err=%v", j, err) + } + // 进程重启后:恢复器不能自作主张把半写目标对外开放。 + if err = e.Recover(context.Background()); err == nil { + t.Fatal("recover reopened a target with unverified content") + } + for _, s := range c.services { + if s.Running { + t.Fatalf("recover restarted %s", s.ID) + } + } + // 原作业接管:不允许重新 stop(写入者本来就没起来),也不从零重来。 + adopted := 0 + _, err = e.RunWindow(context.Background(), "job-half", func(context.Context) error { t.Fatal("prepare ran on adoption"); return nil }, + func(ctx context.Context, w *Window) (string, error) { adopted++; return "", nil }, func(error) bool { return true }) + if err != nil || adopted != 1 { + t.Fatalf("adopt err=%v adopted=%d", err, adopted) + } + for _, s := range c.services { + if !s.Running { + t.Fatalf("writer %s was not restored after adoption", s.ID) + } + } + if _, statErr := os.Stat(root + "/job-half.json"); statErr != nil { + t.Fatal(statErr) + } +} + +// 写到一半进程被杀:journal 必须记住"内容有风险",重启恢复不得放行。 +func TestWindowCrashDuringWriteKeepsTargetHeld(t *testing.T) { + root, c, e := instanceFixture(t) + e.ingressClosed = func(context.Context) error { return nil } + _, err := e.RunWindow(context.Background(), "job-kill", func(context.Context) error { return nil }, + func(ctx context.Context, w *Window) (string, error) { + if err := w.MarkContentRisk(ctx); err != nil { + t.Fatal(err) + } + return "", errors.New("worker 被 kill") + }, func(error) bool { return false }) + if err == nil { + t.Fatal("kill scenario reported success") + } + j, err := e.load("job-kill") + if err != nil || !j.ContentRisk || j.State != "held" { + t.Fatalf("journal=%+v err=%v", j, err) + } + // 重新打开(模拟进程重启):磁盘上的证据必须足以阻止自动开放。 + reopened, err := Open(root, "target-instance", c) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if err = reopened.Recover(context.Background()); err == nil { + t.Fatal("recovery released a target with unverified content") + } + if len(c.calls) != 2 { + t.Fatalf("calls=%v", c.calls) + } +} + +// 停止途中崩溃:journal 处于 stopping(内容还没风险),接管方可以继续停, +// 但不需要"重新准备",也不许把没停完的服务当成已停。 +func TestWindowAdoptsStoppingJournal(t *testing.T) { + _, c, e := instanceFixture(t) + // 手工构造一个"停到一半"的现场:app 已停、store 还在跑。 + e.control.(*exitControl).services[0].Running = false + j := journal{Version: 1, Source: "target-instance", Job: "job-mid", Level: "instance", + Ingress: "http://127.0.0.1:1/health", Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()} + j.Services = []serviceRecord{ + {Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true}, + {Service: Service{ID: "store", Running: true, Healthy: true}}, + } + if err := e.save(j); err != nil { + t.Fatal(err) + } + _, err := e.RunWindow(context.Background(), "job-mid", func(context.Context) error { t.Fatal("prepare replayed"); return nil }, + func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true }) + if err != nil { + t.Fatal(err) + } + if strings.Join(c.calls, ",") != "stop:store,start:app,start:store" { + t.Fatalf("calls=%v", c.calls) + } +} + +// 只锁库的层级不许带入口:那会把"没挡入口"说成"挡了"。 +func TestLibraryLevelRejectsIngressClaim(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + c := &fixtureControl{services: []Service{{ID: "app", Running: true, Healthy: true}}} + e, err := Open(root, "target-instance", c) + if err != nil { + t.Fatal(err) + } + defer e.Close() + if err = e.SetTarget("library", "http://127.0.0.1:1/health", nil, nil); err == nil { + t.Fatal("library level accepted an ingress claim") + } + if err = e.SetTarget("instance", "", nil, nil); err == nil { + t.Fatal("instance level accepted an empty ingress") + } + if err = e.SetTarget("", "", nil, nil); err == nil { + t.Fatal("empty level accepted") + } +} diff --git a/internal/librarymaintenance/target.go b/internal/librarymaintenance/target.go new file mode 100644 index 0000000..8f31af6 --- /dev/null +++ b/internal/librarymaintenance/target.go @@ -0,0 +1,252 @@ +// Package librarymaintenance 把"目标维护窗口"落成一个可验证的实例级动作: +// 关闭目标入口 → 停下全部声明的写入者 → 验证在途排空 → 交给发布链路写目标 +// → 恢复原本在跑的服务与入口。 +// +// 它不产生授权、不读生产凭据,也不把"只锁了治理库 scope"当成系统维护。 +package librarymaintenance + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path/filepath" + "sort" + "strings" + "time" + + "knowledge-governance/internal/libraryfreeze" +) + +// LevelInstance 表示写入者被共享给整个实例:停它影响所有知识库, +// 因此窗口必须按"实例级维护"对外标注,而不是"某个库在维护"。 +const LevelInstance = "instance" + +// Target 是目标的精确身份:实例、入口、写入者、要写的库/索引/桶。 +// Fingerprint 覆盖全部字段,用来发现"配置换了但作业还按旧目标跑"。 +type Target struct { + Instance string `json:"instance"` + Level string `json:"level"` + Project string `json:"project,omitempty"` + Writers []string `json:"writers,omitempty"` + Ingress string `json:"ingress,omitempty"` + Database string `json:"database"` + Index string `json:"index"` + Bucket string `json:"bucket"` + Fingerprint string `json:"fingerprint,omitempty"` +} + +// Config 来自部署注入的配置。任何一项缺失都不许"降级成库级维护"蒙过去。 +type Config struct { + Instance string + Project string + Writers []string + NonWriters map[string]string + DockerSocket string + IngressURL string + Database string + Index string + Bucket string + Timeout time.Duration +} + +type Manager struct { + target Target + source string + docker string + writers []string + nonWriters map[string]string + timeout time.Duration + client *http.Client +} + +func New(cfg Config) (*Manager, error) { + instance := strings.TrimSpace(cfg.Instance) + project := strings.TrimSpace(cfg.Project) + ingress := strings.TrimSpace(cfg.IngressURL) + if instance == "" || project == "" { + return nil, errors.New("实例级维护配置缺少 LIBRARY_TARGET_INSTANCE/PROJECT") + } + if len(cfg.Writers) == 0 { + return nil, errors.New("实例级维护配置缺少写入者声明 LIBRARY_TARGET_WRITERS") + } + if !filepath.IsAbs(cfg.DockerSocket) { + return nil, errors.New("Docker socket 必须是绝对路径") + } + parsed, err := url.Parse(ingress) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.Path == "" || parsed.User != nil { + return nil, errors.New("目标入口地址无效(LIBRARY_TARGET_INGRESS_URL)") + } + if cfg.Database == "" || cfg.Index == "" || cfg.Bucket == "" { + return nil, errors.New("目标身份不完整:需要数据库/索引/存储桶") + } + if cfg.Timeout <= 0 { + cfg.Timeout = 2 * time.Minute + } + writers := append([]string(nil), cfg.Writers...) + sort.Strings(writers) + target := Target{Instance: instance, Level: LevelInstance, Project: project, Writers: writers, + Ingress: ingress, Database: cfg.Database, Index: cfg.Index, Bucket: cfg.Bucket} + fingerprint, err := fingerprintOf(target) + if err != nil { + return nil, err + } + target.Fingerprint = fingerprint + return &Manager{target: target, source: project, docker: cfg.DockerSocket, + writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout, + client: &http.Client{Timeout: 5 * time.Second}}, nil +} + +func fingerprintOf(t Target) (string, error) { + canonical := struct { + Instance string `json:"instance"` + Level string `json:"level"` + Project string `json:"project"` + Writers []string `json:"writers"` + Ingress string `json:"ingress"` + Database string `json:"database"` + Index string `json:"index"` + Bucket string `json:"bucket"` + }{t.Instance, t.Level, t.Project, t.Writers, t.Ingress, t.Database, t.Index, t.Bucket} + raw, err := json.Marshal(canonical) + if err != nil { + return "", err + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]), nil +} + +func (m *Manager) Target() Target { return m.target } + +// Description 是写进维护记录的一句话身份,含指纹,便于事后核对"当时动的是谁"。 +func (m *Manager) Description() string { + t := m.target + return fmt.Sprintf("level=%s instance=%s project=%s writers=%s ingress=%s db=%s index=%s bucket=%s fingerprint=%s", + t.Level, t.Instance, t.Project, strings.Join(t.Writers, "+"), t.Ingress, t.Database, t.Index, t.Bucket, t.Fingerprint) +} + +// Open 返回绑定到该目标的协调器与控制器。调用方负责 Close。 +func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.ProjectController, error) { + control, err := libraryfreeze.NewProjectController(m.docker, m.source, m.writers, m.nonWriters, m.timeout) + if err != nil { + return nil, nil, err + } + coordinator, err := libraryfreeze.Open(root, m.source, control) + if err != nil { + control.Close() + return nil, nil, err + } + if err := coordinator.SetTarget(LevelInstance, m.target.Ingress, m.ingressClosed, m.ingressReady); err != nil { + coordinator.Close() + control.Close() + return nil, nil, err + } + return coordinator, control, nil +} + +// OriginalState 是"动之前"的原服务状态,落在维护记录里: +// 只按它恢复,绝不顺手把原本没跑的服务拉起来。 +func (m *Manager) OriginalState(ctx context.Context) (string, error) { + control, err := libraryfreeze.NewProjectController(m.docker, m.source, m.writers, m.nonWriters, m.timeout) + if err != nil { + return "", err + } + defer control.Close() + items, err := control.Inventory(ctx) + if err != nil { + return "", err + } + parts := make([]string, 0, len(items)) + for _, item := range items { + parts = append(parts, fmt.Sprintf("%s=%s running=%t healthy=%t", m.serviceOf(item.ID), item.ID, item.Running, item.Healthy)) + } + sort.Strings(parts) + return strings.Join(parts, "; "), nil +} + +func (m *Manager) serviceOf(id string) string { + return "container:" + id +} + +// VerifyRunning 确认目标入口真的在服务(部署与恢复后都要看,不看容器状态)。 +func (m *Manager) VerifyRunning(ctx context.Context) error { + return m.ingressReady(ctx) +} + +// ingressClosed:入口关掉的证据是"连不上",不是"返回错误页"—— +// 502/503 说明还有一层代理在对外服务,那不是维护窗口。 +func (m *Manager) ingressClosed(ctx context.Context) error { + deadline := time.Now().Add(m.timeout) + var last error + for { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.target.Ingress, nil) + if err != nil { + return err + } + resp, err := m.client.Do(req) + if err != nil { + return nil + } + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + resp.Body.Close() + last = fmt.Errorf("入口仍返回 HTTP %d", resp.StatusCode) + if time.Now().After(deadline) { + return last + } + select { + case <-ctx.Done(): + return last + case <-time.After(200 * time.Millisecond): + } + } +} + +// ingressReady:入口回来 = 真的能返回 200,而不是"容器起来了"。 +func (m *Manager) ingressReady(ctx context.Context) error { + deadline := time.Now().Add(m.timeout) + var last error + for { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.target.Ingress, nil) + if err != nil { + return err + } + resp, err := m.client.Do(req) + if err == nil { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return nil + } + last = fmt.Errorf("入口返回 HTTP %d:%s", resp.StatusCode, strings.TrimSpace(string(body))) + } else { + last = err + } + if time.Now().After(deadline) { + return fmt.Errorf("目标入口没有恢复:%w", last) + } + select { + case <-ctx.Done(): + return last + case <-time.After(500 * time.Millisecond): + } + } +} + +// DatabaseIdentity 把连接串收敛成"host:port/db"这种可以写进维护记录的身份: +// 维护记录里不落库凭据,但必须能说清动的是哪个库。 +func DatabaseIdentity(raw string) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed.Host == "" { + return "", errors.New("目标数据库连接串无效") + } + database := strings.TrimPrefix(parsed.Path, "/") + if database == "" { + return "", errors.New("目标数据库连接串缺少库名") + } + return parsed.Host + "/" + database, nil +} diff --git a/internal/librarymaintenance/target_test.go b/internal/librarymaintenance/target_test.go new file mode 100644 index 0000000..69ad5b2 --- /dev/null +++ b/internal/librarymaintenance/target_test.go @@ -0,0 +1,134 @@ +package librarymaintenance + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func baseConfig() Config { + return Config{ + Instance: "kgmaint-weknora", Project: "kgmaint-weknora", + Writers: []string{"app"}, + NonWriters: map[string]string{"postgres": "database store; not an initiator"}, + DockerSocket: "/var/run/docker.sock", + IngressURL: "http://127.0.0.1:18080/health", + Database: "127.0.0.1:25432/kgmaint_target", + Index: "kgmaint_1028a_chunks", Bucket: "kgmaint-1028a", + } +} + +// 只锁治理库 scope 的"维护"不许被配置成实例级:那会冒充系统维护。 +func TestNewRequiresExactInstanceIdentity(t *testing.T) { + for name, mutate := range map[string]func(*Config){ + "no-instance": func(c *Config) { c.Instance = "" }, + "no-project": func(c *Config) { c.Project = "" }, + "no-writers": func(c *Config) { c.Writers = nil }, + "no-ingress": func(c *Config) { c.IngressURL = "" }, + "bad-ingress": func(c *Config) { c.IngressURL = "ftp://host" }, + "no-index": func(c *Config) { c.Index = "" }, + "no-bucket": func(c *Config) { c.Bucket = "" }, + } { + t.Run(name, func(t *testing.T) { + cfg := baseConfig() + mutate(&cfg) + if _, err := New(cfg); err == nil { + t.Fatal("incomplete target accepted") + } + }) + } +} + +// 指纹必须随目标身份变化:换了索引/桶/入口就是另一个目标。 +func TestFingerprintTracksTargetIdentity(t *testing.T) { + base, err := New(baseConfig()) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(base.Description(), base.Target().Fingerprint) { + t.Fatal("description omits fingerprint") + } + for name, mutate := range map[string]func(*Config){ + "index": func(c *Config) { c.Index = "other_chunks" }, + "bucket": func(c *Config) { c.Bucket = "other-bucket" }, + "ingress": func(c *Config) { c.IngressURL = "http://127.0.0.1:18081/health" }, + "writers": func(c *Config) { c.Writers = []string{"app", "worker"} }, + } { + t.Run(name, func(t *testing.T) { + cfg := baseConfig() + mutate(&cfg) + other, err := New(cfg) + if err != nil { + t.Fatal(err) + } + if other.Target().Fingerprint == base.Target().Fingerprint { + t.Fatalf("fingerprint did not change for %s", name) + } + }) + } +} + +// 入口"关了"必须是连不上;代理还返回 502 不算关。 +func TestIngressClosedRequiresConnectionRefused(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + cfg := baseConfig() + cfg.IngressURL = server.URL + "/health" + manager, err := New(cfg) + if err != nil { + t.Fatal(err) + } + manager.timeout = 400 * time.Millisecond + if err = manager.ingressClosed(context.Background()); err == nil { + t.Fatal("a still-serving proxy was treated as a closed ingress") + } + server.Close() + manager.client.CloseIdleConnections() + if err = manager.ingressClosed(context.Background()); err != nil { + t.Fatalf("closed ingress rejected: %v", err) + } +} + +// 入口"回来了"必须是 200;容器 running 不算。 +func TestIngressReadyRequiresHealthyResponse(t *testing.T) { + status := http.StatusServiceUnavailable + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + })) + defer server.Close() + cfg := baseConfig() + cfg.IngressURL = server.URL + "/health" + manager, err := New(cfg) + if err != nil { + t.Fatal(err) + } + manager.timeout = 300 * time.Millisecond + if err = manager.VerifyRunning(context.Background()); err == nil { + t.Fatal("unhealthy ingress reported ready") + } + status = http.StatusOK + if err = manager.VerifyRunning(context.Background()); err != nil { + t.Fatalf("healthy ingress rejected: %v", err) + } +} + +// 维护记录里的库身份不含凭据。 +func TestDatabaseIdentityDropsCredentials(t *testing.T) { + identity, err := DatabaseIdentity("postgres://user:secret@127.0.0.1:25432/kgmaint_target?sslmode=disable") + if err != nil { + t.Fatal(err) + } + if identity != "127.0.0.1:25432/kgmaint_target" { + t.Fatalf("identity=%q", identity) + } + if strings.Contains(identity, "secret") { + t.Fatal("credentials leaked into identity") + } + if _, err = DatabaseIdentity("not-a-dsn"); err == nil { + t.Fatal("invalid DSN accepted") + } +} diff --git a/internal/platform/postgres/library_maintenance.go b/internal/platform/postgres/library_maintenance.go index 4fa3496..7d5dfd2 100644 --- a/internal/platform/postgres/library_maintenance.go +++ b/internal/platform/postgres/library_maintenance.go @@ -2,6 +2,7 @@ package postgres import ( "context" + "encoding/json" "errors" "fmt" "time" @@ -10,10 +11,18 @@ import ( ) // LibraryMaintenance 是目标维护窗口的持有记录:一个 scope 只能有一个持有者。 +// +// Level 说明这次维护到底动了什么: +// - "instance":入口与写入者被真的关掉(实例级),所有库都受影响; +// - "library":只锁了治理库的一个 scope,目标入口/后台写入没被挡,不能当成系统维护。 type LibraryMaintenance struct { Scope string `json:"scope"` OwnerJobID string `json:"owner_job_id"` TargetIdentity string `json:"target_identity"` + Level string `json:"level"` + TargetFingerprint string `json:"target_fingerprint,omitempty"` + Services []string `json:"services,omitempty"` + IngressState string `json:"ingress_state"` State string `json:"state"` OriginalServiceState string `json:"original_service_state"` Note string `json:"note"` @@ -25,22 +34,48 @@ type LibraryMaintenance struct { // ErrMaintenanceHeld 表示目标维护窗口被别的作业占着:任何新的写入都必须拒绝。 var ErrMaintenanceHeld = errors.New("目标正在维护中,被其他作业占用") +const maintenanceColumns = `scope, owner_job_id, target_identity, level, target_fingerprint, services, + ingress_state, state, original_service_state, note, acquired_at, updated_at, released_at` + // AcquireLibraryMaintenance 原子地拿到维护权: // 同一作业可以重入(接管后继续),别的作业一律拒绝——这不是"尽力而为"的锁。 func (s *Store) AcquireLibraryMaintenance(ctx context.Context, item LibraryMaintenance, now time.Time) (LibraryMaintenance, error) { + level := item.Level + if level == "" { + level = "library" + } + ingress := item.IngressState + if ingress == "" { + ingress = "unknown" + } + services := item.Services + if services == nil { + services = []string{} + } + raw, err := json.Marshal(services) + if err != nil { + return LibraryMaintenance{}, fmt.Errorf("acquire library maintenance: %w", err) + } row := s.db.QueryRow(ctx, ` -INSERT INTO library_maintenance (scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at) -VALUES ($1,$2,$3,$4,$5,$6,$7,$7) +INSERT INTO library_maintenance (scope, owner_job_id, target_identity, level, target_fingerprint, services, + ingress_state, state, original_service_state, note, acquired_at, updated_at) +VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$11) ON CONFLICT (scope) DO UPDATE SET owner_job_id = EXCLUDED.owner_job_id, target_identity = EXCLUDED.target_identity, + level = EXCLUDED.level, + target_fingerprint = EXCLUDED.target_fingerprint, + services = EXCLUDED.services, + ingress_state = EXCLUDED.ingress_state, state = EXCLUDED.state, + original_service_state = EXCLUDED.original_service_state, note = EXCLUDED.note, updated_at = EXCLUDED.updated_at, released_at = NULL WHERE library_maintenance.owner_job_id = EXCLUDED.owner_job_id -RETURNING scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at, released_at`, - item.Scope, item.OwnerJobID, item.TargetIdentity, item.State, item.OriginalServiceState, item.Note, now.UTC()) +RETURNING `+maintenanceColumns, + item.Scope, item.OwnerJobID, item.TargetIdentity, level, item.TargetFingerprint, string(raw), + ingress, item.State, item.OriginalServiceState, item.Note, now.UTC()) acquired, err := scanMaintenance(row) if errors.Is(err, pgx.ErrNoRows) { return LibraryMaintenance{}, ErrMaintenanceHeld @@ -64,6 +99,23 @@ WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, return nil } +// SetLibraryMaintenanceIngress 记录入口的真实状态(closed/open), +// 页面据此区分"入口关了"和"只是容器停了"。 +func (s *Store) SetLibraryMaintenanceIngress(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error { + if state != "closed" && state != "open" { + return errors.New("invalid ingress state") + } + tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET ingress_state=$3, note=$4, updated_at=$5 +WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, state, note, now.UTC()) + if err != nil { + return fmt.Errorf("update library maintenance ingress: %w", err) + } + if tag.RowsAffected() == 0 { + return errors.New("library maintenance not held by this job") + } + return nil +} + // ReleaseLibraryMaintenance 只在完整验证通过后调用;失败路径必须保持窗口。 func (s *Store) ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error { tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET state='released', released_at=$3, updated_at=$3 @@ -79,8 +131,7 @@ WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, // LibraryMaintenanceOf 读当前窗口状态(页面与诊断用)。 func (s *Store) LibraryMaintenanceOf(ctx context.Context, scope string) (*LibraryMaintenance, error) { - row := s.db.QueryRow(ctx, `SELECT scope, owner_job_id, target_identity, state, original_service_state, note, -acquired_at, updated_at, released_at FROM library_maintenance WHERE scope=$1`, scope) + row := s.db.QueryRow(ctx, `SELECT `+maintenanceColumns+` FROM library_maintenance WHERE scope=$1`, scope) item, err := scanMaintenance(row) if errors.Is(err, pgx.ErrNoRows) { return nil, nil @@ -93,7 +144,17 @@ acquired_at, updated_at, released_at FROM library_maintenance WHERE scope=$1`, s func scanMaintenance(row pgx.Row) (LibraryMaintenance, error) { var item LibraryMaintenance - err := row.Scan(&item.Scope, &item.OwnerJobID, &item.TargetIdentity, &item.State, - &item.OriginalServiceState, &item.Note, &item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt) - return item, err + var raw []byte + err := row.Scan(&item.Scope, &item.OwnerJobID, &item.TargetIdentity, &item.Level, &item.TargetFingerprint, + &raw, &item.IngressState, &item.State, &item.OriginalServiceState, &item.Note, + &item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt) + if err != nil { + return LibraryMaintenance{}, err + } + if len(raw) > 0 { + if err := json.Unmarshal(raw, &item.Services); err != nil { + return LibraryMaintenance{}, fmt.Errorf("maintenance services unreadable: %w", err) + } + } + return item, nil } diff --git a/internal/platform/postgres/migrations/000071_library_maintenance_level.down.sql b/internal/platform/postgres/migrations/000071_library_maintenance_level.down.sql new file mode 100644 index 0000000..1a6df8f --- /dev/null +++ b/internal/platform/postgres/migrations/000071_library_maintenance_level.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE library_maintenance + DROP COLUMN IF EXISTS ingress_state, + DROP COLUMN IF EXISTS services, + DROP COLUMN IF EXISTS target_fingerprint, + DROP COLUMN IF EXISTS level; \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000071_library_maintenance_level.up.sql b/internal/platform/postgres/migrations/000071_library_maintenance_level.up.sql new file mode 100644 index 0000000..b8d45fa --- /dev/null +++ b/internal/platform/postgres/migrations/000071_library_maintenance_level.up.sql @@ -0,0 +1,9 @@ +-- 维护记录要能区分"实例级维护"(入口与写入者被真的关掉)与"仅治理库互斥"。 +-- 也要能事后核对:动的是哪个实例、哪几个容器、入口当时是不是真的关了。 +ALTER TABLE library_maintenance + ADD COLUMN level VARCHAR(16) NOT NULL DEFAULT 'library' + CHECK (level IN ('library', 'instance')), + ADD COLUMN target_fingerprint VARCHAR(64) NOT NULL DEFAULT '', + ADD COLUMN services JSONB NOT NULL DEFAULT '[]'::jsonb, + ADD COLUMN ingress_state VARCHAR(16) NOT NULL DEFAULT 'unknown' + CHECK (ingress_state IN ('unknown', 'closed', 'open')); \ No newline at end of file diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index eaa843a..1aceceb 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,25 +11,30 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 70 { - t.Fatalf("migration count = %d, want 70", len(migrations)) + if len(migrations) != 71 { + t.Fatalf("migration count = %d, want 71", len(migrations)) } - if tail := migrations[len(migrations)-2]; tail.Version != 69 || + // 71 让维护记录能区分实例级维护与仅治理库互斥。 + if level := migrations[len(migrations)-1]; level.Version != 71 || + !strings.Contains(level.UpSQL, "level") || !strings.Contains(level.UpSQL, "ingress_state") { + t.Fatalf("migration 71 = %#v", level) + } + if maint := migrations[len(migrations)-2]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") { + t.Fatalf("migration 70 = %#v", maint) + } + if tail := migrations[len(migrations)-3]; tail.Version != 69 || !strings.Contains(tail.UpSQL, "library_jobs_write_target") { t.Fatalf("migration 69 = %#v", tail) } - if maint := migrations[len(migrations)-1]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") { - t.Fatalf("migration 70 = %#v", maint) - } - if restore := migrations[len(migrations)-3]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { + if restore := migrations[len(migrations)-4]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { t.Fatalf("migration 68 = %#v", restore) } - if last := migrations[len(migrations)-4]; last.Version != 67 || + if last := migrations[len(migrations)-5]; last.Version != 67 || !strings.Contains(last.UpSQL, "library_release_authorizations") || !strings.Contains(last.UpSQL, "params_digest") { t.Fatalf("migration 67 = %#v", last) } - if flow := migrations[len(migrations)-5]; flow.Version != 66 || + if flow := migrations[len(migrations)-6]; flow.Version != 66 || !strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") || !strings.Contains(flow.UpSQL, "library_jobs_active") { t.Fatalf("migration 66 = %#v", flow)