feat(library): 同步/恢复改为实例级维护窗口
build / build (push) Successful in 1m26s

只锁治理库 scope 挡不住 WeKnora 直连和后台写入。把封存链路的
维护机制接到发布路径:持久化 owner/目标身份/原服务状态,关入口、
停写入者、验证排空与入口不可达,再写目标;写完先恢复原运行状态
与入口才释放占用。内容风险一旦落盘,重启恢复不再自动启动写入者。

新增 000071 迁移记录 level/指纹/服务/入口状态;未接线的目标直接
拒绝写,不用库级锁冒充系统维护。
This commit is contained in:
姚章浩
2026-10-28 23:01:29 +08:00
parent fe6e218d9c
commit 6fe98c9a30
14 changed files with 1525 additions and 238 deletions
+32
View File
@@ -1,5 +1,37 @@
# Changelog
## 开发快照(目标维护:实例级窗口)
### Added
- 同步/恢复不再只锁治理库 scope:`runLibraryRelease` 现在先开**实例级维护窗口**
(`internal/librarymaintenance` + `internal/libraryfreeze` 复用)——
持久化 owner / 目标精确身份(实例、写入者容器、入口、库/索引/桶 + 指纹)/ 原服务状态,
关闭目标入口(SIGTERM 后监听立即关闭)、停下全部声明写入者,
并验证「写入者真的停了」「在途请求真的排空了」「入口真的连不上」之后才允许写目标。
- `libraryfreeze.RunWindow`:一个窗口里跑调用方的写阶段。第一次写目标前必须
`MarkContentRisk`(fsync 落盘);写完先恢复原运行状态与入口,再释放维护占用。
- 入口探针(`internal/librarymaintenance`):关闭 = 连不上(代理还回 502 不算关闭),
恢复 = HTTP 200;只有原本在跑的服务会被重启。
- 迁移 `000071_library_maintenance_level`:维护记录新增 `level`(instance/library)、
`target_fingerprint`、`services`、`ingress_state`,页面/审计能看清这次到底动的是实例还是只锁了一个库。
### Fixed
- 进程被杀/授权过期不再误放行半写目标:journal 里的 `content_risk` 一旦置位,
重启恢复只收尾确定性的事,**不会**自动把写入者启动回来;原作业接管后才继续。
- 回滚失败(`rollback_failed`)或校验失败(`finalize` 不一致)时,服务保持停止、
维护占用保持持有(`needs_restore`),入口不重新开放。
- 接管场景(已有预备份)一律按「内容可能半写」处理,任何早退都不重新开放目标。
### 限制(本轮不承诺)
- 维护层级判定是「配置里声明的写入者组」:多实例/多写入者拓扑变更需要重新声明,
不会自动猜测新容器。
- 前端只透传新增字段,页面尚未为 `level/ingress_state` 做专门的文案与布局;
Wiki 语义重算与浏览器整体验收仍未完成。
- 本机专用候选实例(kgmaint-*)仅承载验收数据,不代表远程生产已验证。
## 开发快照(读接口超时:真实修复)
### Fixed
+266 -103
View File
@@ -12,11 +12,49 @@ import (
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/librarymaintenance"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// 目标内容在窗口内的几种"已知状态",决定目标能否被重新开放:
// - contentUntouched:这次窗口还没写过目标(可以安全开放)
// - contentSucceeded / contentRolledBack:内容已验证,或已回滚并验证(可以开放)
// - contentHalfWritten / contentRollbackFails:内容可能半写(必须保持维护)
const (
contentUntouched = ""
contentSucceeded = "succeeded"
contentRolledBack = "rolled_back"
contentHalfWritten = "half_written"
contentRollbackFails = "rollback_failed"
)
// contentTrackedTarget 只做一件事:在第一次写目标时把"内容有风险"标出来。
// 这样"授权过期/进程被杀/校验失败"这些早退路径不会把半写目标当成没动过。
type contentTrackedTarget struct {
inner libraryrelease.Target
onApply func()
}
func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) {
return t.inner.Current(ctx)
}
func (t *contentTrackedTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error {
t.onApply()
return t.inner.Apply(ctx, stage, snapshotID)
}
func (t *contentTrackedTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error {
return t.inner.Verify(ctx, expect)
}
func (t *contentTrackedTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error {
return t.inner.Restore(ctx, backup)
}
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
type libraryReleasePayload struct {
LibraryJobID string `json:"library_job_id"`
@@ -27,7 +65,28 @@ type libraryReleasePayload struct {
FinalizeOnly bool `json:"finalize_only,omitempty"`
}
// runLibraryRelease 执行"闸口 → 授权 → 预备份 → 四类写入 → 校验"的完整链路。
// openLibraryMaintenance 按部署注入的配置装配实例级维护目标。
// 未接线时返回明确原因:只锁治理库 scope 不算维护,宁可拒绝写也不冒充。
func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) {
database, err := librarymaintenance.DatabaseIdentity(config.LibraryProductionDatabaseURL)
if err != nil {
return nil, err
}
return librarymaintenance.New(librarymaintenance.Config{
Instance: config.LibraryTargetInstance,
Project: config.LibraryTargetProject,
Writers: config.LibraryTargetWriters,
NonWriters: config.LibraryTargetNonWriters,
DockerSocket: config.LibraryTargetDockerSocket,
IngressURL: config.LibraryTargetIngressURL,
Database: database,
Index: config.LibraryProductionIndex,
Bucket: config.LibraryProductionS3Bucket,
})
}
// runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 →
// 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。
// 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。
func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config appconfig.Config,
job jobs.Job, kind governance.LibraryJobKind) error {
@@ -106,134 +165,238 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
}
target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping)
if wireErr == nil {
if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
recorder.mappingDigest = digest
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
recorder.mappingDigest = mapping
}
}
if wireErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup")
}
defer target.Close()
// 维护窗口:目标在同步/恢复期间被本作业独占,任何其他写入一律拒绝。
// 维护目标必须在真正动目标之前确定:缺配置就拒绝写,不用"库级锁"顶替。
manager, managerErr := openLibraryMaintenance(config)
if managerErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup")
}
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
originalState, stateErr := manager.OriginalState(ctx)
if stateErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup")
}
identity := manager.Target()
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity,
State: "acquired", OriginalServiceState: "candidate: no external writer configured",
Note: "同步/恢复期间目标维护,验证通过才开放",
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID,
TargetIdentity: targetIdentity + " | " + manager.Description(),
Level: librarymaintenance.LevelInstance,
TargetFingerprint: identity.Fingerprint,
Services: identity.Writers,
IngressState: "unknown",
State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
}, now)
if acquireErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup")
}
_ = maintenance
if payload.FinalizeOnly {
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil {
message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
// 协调器都打不开:这一次没动过目标。接管场景除外——上一次尝试可能已经把
// 写入者停住并写了一半,那种情况下保持占用是唯一安全的选择。
if resumed {
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
} else {
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
"未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC())
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC())
}
mappingDigest, _ := target.Search().MappingSHA256(ctx)
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker",
Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only",
"index_mapping_sha256": mappingDigest},
CreatedAt: time.Now().UTC(),
}); err != nil {
message := "收尾续作仍无法登记生效版本:" + err.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
// 只有校验+登记都完成才开放目标。
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC())
return nil
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "维护协调器不可用:"+openErr.Error(), "prebackup")
}
defer control.Close()
defer coordinator.Close()
var backupSource libraryrelease.BackupStore = backups
// 接管场景下目标可能已经被上一次尝试写过:在证明"已回滚或已验证"之前,
// 一律按半写对待,任何早退都不许重新开放目标。
contentState := contentUntouched
if resumed {
backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
SnapshotID: previous.SnapshotID}}
contentState = contentHalfWritten
}
tracked := &contentTrackedTarget{inner: target, onApply: func() { contentState = contentHalfWritten }}
allowRestore := func(error) bool {
return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack
}
request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID,
ParamsDigest: digest, Now: now,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
case "succeeded":
mappingDigest := ""
if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
mappingDigest = digest
}
// 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。
// 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。
recorder.Stage(ctx, "finalize", "running", "登记生产生效版本")
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
Scope: payload.Scope,
Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID,
SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy,
Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID,
"backup_bytes": outcome.Backup.Bytes, "action": string(kind),
"index_mapping_sha256": mappingDigest},
CreatedAt: finishedAt,
})
if versionErr != nil {
message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
// 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr)
execute := func(ctx context.Context) error {
if payload.FinalizeOnly {
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
if verifyErr := tracked.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil {
contentState = contentHalfWritten
message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
return errors.New(message)
contentState = contentSucceeded
mappingDigest, _ := target.Search().MappingSHA256(ctx)
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker",
Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only",
"index_mapping_sha256": mappingDigest},
CreatedAt: time.Now().UTC(),
}); err != nil {
message := "收尾续作仍无法登记生效版本:" + err.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return nil
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本")
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
// 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。
return fmt.Errorf("作业状态写入失败: %w", advanceErr)
var backupSource libraryrelease.BackupStore = backups
if resumed {
backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
SnapshotID: previous.SnapshotID}}
}
// 验证通过 + 版本登记成功 = 唯一开放目标的路径。
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, finishedAt)
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID)
return nil
case "rolled_back":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"已回滚到上一版本,但验证未通过:目标继续维护,人工确认后再开放", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"回滚失败:目标保持维护,必须按预备份恢复", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID,
ParamsDigest: digest, Now: now,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
outcome := libraryrelease.Run(ctx, request, tracked, backupSource, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
case "succeeded":
contentState = contentSucceeded
mappingDigest := ""
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
mappingDigest = mapping
}
// 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。
// 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。
recorder.Stage(ctx, "finalize", "running", "登记生产生效版本")
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
Scope: payload.Scope,
Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID,
SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy,
Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID,
"backup_bytes": outcome.Backup.Bytes, "action": string(kind),
"index_mapping_sha256": mappingDigest},
CreatedAt: finishedAt,
})
if versionErr != nil {
message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
// 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr)
}
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本")
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
// 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。
return fmt.Errorf("作业状态写入失败: %w", advanceErr)
}
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID)
return nil
case "rolled_back":
contentState = contentRolledBack
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
contentState = contentRollbackFails
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
default:
recovery := outcome.Recovery
if recovery == "" {
recovery = "未对生产执行任何写入,无需恢复"
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup",
outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt)
return errors.New(outcome.Error)
}
}
prepare := func(ctx context.Context) error {
// 停下写入者之前必须确认目标现在是健康的:拿一个已经不健康的目标开窗口,
// 只会把"本来坏的"记成"维护造成的"。
return manager.VerifyRunning(ctx)
}
windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) {
if err := w.MarkContentRisk(ctx); err != nil {
return "", err
}
if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed",
"目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil {
// 记录失败不影响隔离本身,但必须留痕,不能假装记上了。
log.Printf("library %s maintenance ingress record failed: %v", payload.LibraryJobID, ingressErr)
}
return "", execute(ctx)
}
window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore)
// 收尾:只有"内容已验证/已回滚"且服务与入口都恢复,才释放维护占用。
now2 := time.Now().UTC()
switch {
case window.State == "held":
note := "目标保持维护:内容状态未验证(半写或回滚失败),或服务/入口未能恢复"
if windowErr != nil {
note += ":" + windowErr.Error()
}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
if windowErr == nil {
return errors.New(note)
}
return windowErr
case contentState == contentSucceeded:
_ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "目标入口已恢复并验证", now2)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
"写入与校验完成,目标已按原运行状态恢复", now2)
if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil {
return fmt.Errorf("目标已恢复但维护占用释放失败: %w", err)
}
return windowErr
case contentState == contentRolledBack:
_ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "已回滚并验证,目标入口恢复", now2)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
"本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2)
if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil {
return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err)
}
return windowErr
default:
recovery := outcome.Recovery
if recovery == "" {
recovery = "未对生产执行任何写入,无需恢复"
// 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。
_ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open",
"未进入维护窗口,目标未被停用", now2)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
"未写目标,维护窗口未开启", now2)
if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil {
return fmt.Errorf("维护占用释放失败: %w", err)
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup",
outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt)
return errors.New(outcome.Error)
return windowErr
}
}
+76
View File
@@ -94,6 +94,15 @@ type Config struct {
// LibraryRetentionVersions / Days 是保留策略;都为 0 表示未设置(不生成清理候选)。
LibraryRetentionVersions int
LibraryRetentionDays int
// 目标维护(实例级):同步/恢复期间关闭目标入口、停下写入者、排空在途请求。
// 任一项留空即"未接线",生产写入会被拒绝——只锁治理库 scope 不算维护。
LibraryTargetInstance string
LibraryTargetProject string
LibraryTargetWriters []string
LibraryTargetNonWriters map[string]string
LibraryTargetDockerSocket string
LibraryTargetIngressURL string
LibraryTargetJournalRoot string
// LibraryAgentSocket 是主机侧封存 agent 的 unix socket(只由 worker 连接)。
// 留空即"未接线":页面不显示可用的封存入口,接口返回原因而不是假装已排队。
LibraryAgentSocket string
@@ -182,6 +191,13 @@ func Load(getenv func(string) string) (Config, error) {
LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")),
LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")),
LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")),
LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")),
LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")),
LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")),
LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")),
LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"),
LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")),
LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")),
LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"),
LibraryEmbeddingDimension: embeddingDimension,
LibraryCapacityBudgetBytes: capacityBudget,
@@ -302,6 +318,39 @@ func Load(getenv func(string) string) (Config, error) {
if config.LibraryAgentSocket != "" && !filepath.IsAbs(config.LibraryAgentSocket) {
return Config{}, errors.New("LIBRARY_AGENT_SOCKET 必须是绝对路径(unix socket 由主机侧 agent 持有)")
}
// 目标维护要么完整配置,要么完全不配;半配置会让窗口"看起来存在"却挡不住目标。
targetSet := config.LibraryTargetInstance != "" || config.LibraryTargetProject != "" ||
len(config.LibraryTargetWriters) > 0 || config.LibraryTargetIngressURL != ""
if targetSet {
missing := make([]string, 0, 4)
if config.LibraryTargetInstance == "" {
missing = append(missing, "LIBRARY_TARGET_INSTANCE")
}
if config.LibraryTargetProject == "" {
missing = append(missing, "LIBRARY_TARGET_PROJECT")
}
if len(config.LibraryTargetWriters) == 0 {
missing = append(missing, "LIBRARY_TARGET_WRITERS")
}
if config.LibraryTargetIngressURL == "" {
missing = append(missing, "LIBRARY_TARGET_INGRESS_URL")
}
if config.LibraryTargetJournalRoot == "" {
missing = append(missing, "LIBRARY_TARGET_JOURNAL_ROOT")
}
if len(missing) > 0 {
return Config{}, fmt.Errorf("目标维护配置不完整:缺少 %s", strings.Join(missing, "、"))
}
if !filepath.IsAbs(config.LibraryTargetJournalRoot) {
return Config{}, errors.New("LIBRARY_TARGET_JOURNAL_ROOT 必须是绝对路径")
}
if !filepath.IsAbs(config.LibraryTargetDockerSocket) {
return Config{}, errors.New("LIBRARY_TARGET_DOCKER_SOCKET 必须是绝对路径")
}
if len(config.LibraryTargetNonWriters) == 0 {
return Config{}, errors.New("LIBRARY_TARGET_NON_WRITERS 必须逐项说明项目内非写入服务")
}
}
if config.TestWeKnoraHTTPTimeout, err = durationOrDefault(getenv("TEST_WEKNORA_HTTP_TIMEOUT"), config.TestWeKnoraHTTPTimeout); err != nil {
return Config{}, fmt.Errorf("TEST_WEKNORA_HTTP_TIMEOUT: %w", err)
}
@@ -419,6 +468,33 @@ func splitOrigins(value string) []string {
return origins
}
// splitList 把逗号分隔的声明拆成去空列表(顺序保留,便于日志对账)。
func splitList(value string) []string {
parts := strings.Split(value, ",")
out := make([]string, 0, len(parts))
for _, part := range parts {
if trimmed := strings.TrimSpace(part); trimmed != "" {
out = append(out, trimmed)
}
}
return out
}
// splitDeclarations 解析 "name=reason,name2=reason2"。
// 没有理由的声明直接丢空,由 Validate 拒绝——"忘记声明"不能变成"不用声明"。
func splitDeclarations(value string) map[string]string {
out := map[string]string{}
for _, item := range splitList(value) {
name, reason, ok := strings.Cut(item, "=")
name, reason = strings.TrimSpace(name), strings.TrimSpace(reason)
if !ok || name == "" || reason == "" {
continue
}
out[name] = reason
}
return out
}
// valueOrDefaultInt 解析正整数(非法/缺省都退回默认值)。
func valueOrDefaultInt(value string, fallback int) int {
parsed, err := strconv.Atoi(strings.TrimSpace(value))
+9 -1
View File
@@ -78,6 +78,7 @@ type dockerInspection struct {
Restarting bool
Dead bool
Status string
ExitCode int
Health *struct{ Status string }
} `json:"State"`
}
@@ -104,7 +105,14 @@ func (d *DockerController) inspect(ctx context.Context, id string) (Service, err
return Service{}, errors.New("contradictory test writer state")
}
healthy := data.State.Running && data.State.Health != nil && data.State.Health.Status == "healthy"
return Service{ID: id, Running: data.State.Running, Healthy: healthy}, nil
// 退出码只在"已经停下"时才有意义:它证明服务是自己收尾退出的(0),
// 还是被信号切断的(143/137)。运行中的服务不提供这项证据。
var exit *int
if !data.State.Running {
code := data.State.ExitCode
exit = &code
}
return Service{ID: id, Running: data.State.Running, Healthy: healthy, ExitCode: exit}, nil
}
func (d *DockerController) Inventory(ctx context.Context) ([]Service, error) {
filters, _ := json.Marshal(map[string][]string{"label": {sourceLabel + "=" + d.source}})
+72 -115
View File
@@ -9,7 +9,9 @@ import (
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"sort"
@@ -22,6 +24,9 @@ type Service struct {
ID string `json:"id"`
Running bool `json:"running"`
Healthy bool `json:"healthy"`
// ExitCode 是"停下之后"的进程退出码,只有控制器真的读得到时才填。
// 它是在途请求是否被排空的证据:0 = 服务自己收尾退出,143/137 = 被切断。
ExitCode *int `json:"exit_code,omitempty"`
}
// Inventory must enumerate the complete, independently scoped test writer set,
@@ -47,15 +52,29 @@ type journal struct {
Source string `json:"source"`
Job string `json:"job"`
Result
Services []serviceRecord `json:"services"`
StartedAt time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
Code string `json:"code,omitempty"`
// Level 区分"实例级维护"(写入者/入口被真的关掉)与"仅治理库互斥"。
// 没有 Level 的旧 journal 来自封存链路,按实例级处理但不冒充已验证。
Level string `json:"level,omitempty"`
Ingress string `json:"ingress,omitempty"`
IngressClosed bool `json:"ingress_closed,omitempty"`
DrainVerified bool `json:"drain_verified,omitempty"`
ContentRisk bool `json:"content_risk,omitempty"`
RestoreBlocked bool `json:"restore_blocked,omitempty"`
Services []serviceRecord `json:"services"`
StartedAt time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
Code string `json:"code,omitempty"`
}
type Coordinator struct {
root *os.Root
source string
control Controller
level string
ingress string
// ingressClosed / ingressReady 是可选的外部探针:声明了实例级维护就必须
// 证明入口真的关了、真的回来了,而不是"停了容器就算关"。
ingressClosed func(context.Context) error
ingressReady func(context.Context) error
}
func Open(root, source string, c Controller) (*Coordinator, error) {
@@ -78,6 +97,30 @@ func Open(root, source string, c Controller) (*Coordinator, error) {
}
func (c *Coordinator) Close() error { return c.root.Close() }
// SetTarget 声明这次维护的层级与入口。实例级维护必须指名入口地址;
// 只锁治理 scope 的维护不许带入口,否则就是拿"库级"冒充"实例级"。
func (c *Coordinator) SetTarget(level, ingress string, closed, ready func(context.Context) error) error {
switch level {
case "instance":
if !validIngress(ingress) {
return errors.New("instance maintenance requires the target ingress address")
}
case "library":
if ingress != "" || closed != nil || ready != nil {
return errors.New("library scope lock must not claim ingress control")
}
default:
return errors.New("invalid maintenance level")
}
c.level, c.ingress, c.ingressClosed, c.ingressReady = level, ingress, closed, ready
return nil
}
func validIngress(raw string) bool {
parsed, err := url.Parse(raw)
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.Path != "" && parsed.User == nil
}
func (c *Coordinator) Recover(ctx context.Context) error {
lock, err := c.lock()
if err != nil {
@@ -112,6 +155,13 @@ func (c *Coordinator) Recover(ctx context.Context) error {
pending = append(pending, j)
}
}
// 内容可能已被写过的窗口不能自动开放:重启恢复只许"收尾确定性的事",
// 半写目标必须由原作业接管或人工按预备份恢复,否则就是把半写当成品放行。
for _, j := range pending {
if j.ContentRisk || j.RestoreBlocked {
return fmt.Errorf("维护窗口 %s 的内容状态未验证(可能已半写):目标保持停用,须由原作业接管或按预备份恢复", j.Job)
}
}
if len(pending) > 1 {
return errors.New("multiple interrupted freezes require operator reconciliation")
}
@@ -245,10 +295,23 @@ func (c *Coordinator) load(job string) (journal, error) {
return journal{}, errors.New("freeze journal identity mismatch")
}
switch j.State {
case "stopping", "capturing", "restoring", "succeeded", "failed":
case "stopping", "held", "capturing", "restoring", "succeeded", "failed":
default:
return journal{}, errors.New("invalid freeze journal state")
}
if j.Level != "" && j.Level != "instance" && j.Level != "library" {
return journal{}, errors.New("invalid freeze maintenance level")
}
// 声明了实例级维护就必须留下入口证据;只锁库的窗口不许带入口。
if j.Level == "instance" && !validIngress(j.Ingress) {
return journal{}, errors.New("invalid freeze ingress evidence")
}
if j.Level == "library" && (j.Ingress != "" || j.IngressClosed) {
return journal{}, errors.New("library scope lock claims ingress control")
}
if j.RestoreBlocked && !j.ContentRisk {
return journal{}, errors.New("invalid freeze restore block")
}
if len(j.Services) == 0 || len(j.Services) > 256 {
return journal{}, errors.New("invalid freeze journal services")
}
@@ -256,6 +319,10 @@ func (c *Coordinator) load(job string) (journal, error) {
if !validID(s.ID) || (i > 0 && j.Services[i-1].ID >= s.ID) || s.StopRequested && !s.Running || s.Restored && (!s.StopRequested || !s.Running) {
return journal{}, errors.New("invalid freeze service ownership")
}
// 退出码只能出现在"被要求停过"的服务上;Running 记的是原始状态。
if s.ExitCode != nil && !s.StopRequested {
return journal{}, errors.New("invalid freeze drain evidence")
}
}
return j, nil
}
@@ -379,113 +446,3 @@ func (c *Coordinator) restore(ctx context.Context, j *journal) error {
}
return failures
}
func (c *Coordinator) Run(ctx context.Context, job string, capture func(context.Context) (string, error)) (Result, error) {
return c.RunPrepared(ctx, job, func(context.Context) error { return nil }, capture)
}
// RunPrepared runs readiness checks under the source lock, before any stop
// intent. It does not repeat checks or capture for an already completed job.
func (c *Coordinator) RunPrepared(ctx context.Context, job string, prepare func(context.Context) error, capture func(context.Context) (string, error)) (Result, error) {
if !validID(job) || capture == nil || prepare == nil {
return Result{}, errors.New("invalid freeze request")
}
if err := ctx.Err(); err != nil {
return Result{}, err
}
lock, err := c.lock()
if err != nil {
return Result{}, err
}
defer unlock(lock)
previous, err := c.load(job)
if err == nil {
if previous.State == "succeeded" {
return previous.Result, nil
}
return previous.Result, errors.New("freeze request already exists; recovery or a new request is required")
}
if !errors.Is(err, os.ErrNotExist) {
return Result{}, err
}
if err = c.pending(); err != nil {
return Result{}, err
}
if err = prepare(ctx); err != nil {
return Result{}, err
}
if err = ctx.Err(); err != nil {
return Result{}, err
}
items, err := inventory(ctx, c.control)
if err != nil {
return Result{}, err
}
j := journal{Version: 1, Source: c.source, Job: job, Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
for _, s := range items {
if s.Running && !s.Healthy {
return Result{}, errors.New("test writer is not ready")
}
j.Services = append(j.Services, serviceRecord{Service: s})
}
if err = c.save(j); err != nil {
return Result{}, err
}
var workErr error
for i := range j.Services {
if !j.Services[i].Running {
continue
}
j.Services[i].StopRequested = true
if workErr = c.save(j); workErr != nil {
break
}
if workErr = c.control.Stop(ctx, j.Services[i].ID); workErr != nil {
break
}
}
if workErr == nil {
workErr = c.stopped(ctx, j)
}
if workErr == nil {
for i := range j.Services {
if j.Services[i].StopRequested {
j.Services[i].StopVerified = true
}
}
j.State = "capturing"
workErr = c.save(j)
}
if workErr == nil {
j.SnapshotID, workErr = invokeCapture(ctx, capture)
}
if workErr == nil {
workErr = ctx.Err()
}
if workErr == nil {
decoded, decodeErr := hex.DecodeString(j.SnapshotID)
if decodeErr != nil || len(decoded) != 32 || strings.ToLower(j.SnapshotID) != j.SnapshotID {
workErr = errors.New("invalid candidate snapshot digest")
}
}
if workErr == nil {
workErr = c.stopped(ctx, j)
}
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
defer cancel()
recoveryErr := c.restore(recoveryCtx, &j)
if recoveryErr != nil {
return j.Result, errors.Join(workErr, recoveryErr)
}
if workErr != nil {
j.State = "failed"
j.Code = "capture_or_isolation_failed"
} else {
j.State = "succeeded"
}
ended := time.Now().UTC()
j.FinishedAt = &ended
if err = c.save(j); err != nil {
return j.Result, err
}
return j.Result, workErr
}
+9 -1
View File
@@ -113,6 +113,7 @@ type projectInspection struct {
Restarting bool
Dead bool
Status string
ExitCode int
Health *struct{ Status string }
} `json:"State"`
}
@@ -158,7 +159,14 @@ func (p *ProjectController) inspect(ctx context.Context, id string) (Service, er
return Service{}, errors.New("contradictory project writer state")
}
healthy := data.State.Running && data.State.Health != nil && data.State.Health.Status == "healthy"
return Service{ID: id, Running: data.State.Running, Healthy: healthy}, nil
// 退出码只在"已经停下"时才有意义:它证明服务是自己收尾退出的(0),
// 还是被信号切断的(143/137)。运行中的服务不提供这项证据。
var exit *int
if !data.State.Running {
code := data.State.ExitCode
exit = &code
}
return Service{ID: id, Running: data.State.Running, Healthy: healthy, ExitCode: exit}, nil
}
// Inventory enumerates the complete project writer group and rejects an
+324
View File
@@ -0,0 +1,324 @@
package libraryfreeze
import (
"context"
"encoding/hex"
"errors"
"fmt"
"os"
"strings"
"time"
)
// Window 是一次维护窗口里"写目标"阶段的句柄。
//
// work 必须在第一笔目标写入之前调用 MarkContentRisk:它是一条持久证据,
// 记录"从这里开始目标内容可能已被改动"。进程在这之后死掉时,恢复器不会
// 自动把目标开放——半写目标只能由原作业接管或按预备份恢复。
type Window struct {
c *Coordinator
j *journal
}
func (w *Window) MarkContentRisk(ctx context.Context) error {
if w == nil || w.c == nil || w.j == nil {
return errors.New("invalid maintenance window")
}
if err := ctx.Err(); err != nil {
return err
}
if w.j.ContentRisk {
return nil
}
w.j.ContentRisk = true
return w.c.save(*w.j)
}
// Job 返回窗口所属作业 id(恢复器与调用方据此对账)。
func (w *Window) Job() string {
if w == nil || w.j == nil {
return ""
}
return w.j.Job
}
func (c *Coordinator) Run(ctx context.Context, job string, capture func(context.Context) (string, error)) (Result, error) {
return c.RunPrepared(ctx, job, func(context.Context) error { return nil }, capture)
}
// RunPrepared 是封存链路的入口:停止写入者 → 在窗口内取快照 → 恢复写入者。
// 它等价于 allowRestore 恒为 true 的 RunWindow。
func (c *Coordinator) RunPrepared(ctx context.Context, job string, prepare func(context.Context) error, capture func(context.Context) (string, error)) (Result, error) {
if capture == nil {
return Result{}, errors.New("invalid freeze request")
}
return c.RunWindow(ctx, job, prepare, func(ctx context.Context, _ *Window) (string, error) {
return capture(ctx)
}, func(error) bool { return true })
}
// RunWindow 是维护窗口的唯一实现:
//
// 关入口 + 停全部写入者 → 验证真的停了、在途真的排空了 → 跑 work
// → 恢复"原本在跑"的服务 → 验证入口回来
//
// allowRestore 在 work 失败后决定目标能否重新开放:返回 false 表示内容状态
// 未验证(半写)——此时服务保持停止、journal 保持非终态,崩溃恢复也不许
// 自动启动它们。同一个 job 可以接管未完成的窗口,别的 job 不行。
func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(context.Context) error,
work func(context.Context, *Window) (string, error), allowRestore func(error) bool) (Result, error) {
if !validID(job) || work == nil || prepare == nil {
return Result{}, errors.New("invalid freeze request")
}
if allowRestore == nil {
allowRestore = func(error) bool { return true }
}
if err := ctx.Err(); err != nil {
return Result{}, err
}
lock, err := c.lock()
if err != nil {
return Result{}, err
}
defer unlock(lock)
previous, loadErr := c.load(job)
j := journal{}
switch {
case loadErr == nil && previous.State == "succeeded":
return previous.Result, nil
case loadErr == nil && (previous.State == "stopping" || previous.State == "held"):
// 接管:目标必须仍然停住。不重新"准备"、也不从零重来。
j = previous
if j.State == "held" && j.RestoreBlocked {
// 上一次窗口把内容判成半写:接管方必须自己重新证明,不许清掉风险。
j.RestoreBlocked = false
}
case loadErr == nil:
return previous.Result, errors.New("freeze request already exists; recovery or a new request is required")
case !errors.Is(loadErr, os.ErrNotExist):
return Result{}, loadErr
default:
if err = c.pending(); err != nil {
return Result{}, err
}
if err = prepare(ctx); err != nil {
return Result{}, err
}
if err = ctx.Err(); err != nil {
return Result{}, err
}
items, err := inventory(ctx, c.control)
if err != nil {
return Result{}, err
}
j = journal{Version: 1, Source: c.source, Job: job, Level: c.level, Ingress: c.ingress,
Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
for _, s := range items {
if s.Running && !s.Healthy {
return Result{}, errors.New("test writer is not ready")
}
j.Services = append(j.Services, serviceRecord{Service: s})
}
if err = c.save(j); err != nil {
return Result{}, err
}
}
// 停写入者 + 排空 + 关入口:任何一步不成立都不许往下写。
if stopErr := c.stopAll(ctx, &j); stopErr != nil {
return c.abandonWindow(ctx, &j, stopErr, "stop_failed")
}
if stopErr := c.stopped(ctx, j); stopErr != nil {
return c.abandonWindow(ctx, &j, stopErr, "isolation_unverified")
}
if stopErr := c.verifyDrain(ctx, &j); stopErr != nil {
return c.abandonWindow(ctx, &j, stopErr, "drain_unverified")
}
if c.ingressClosed != nil {
if stopErr := c.ingressClosed(ctx); stopErr != nil {
return c.abandonWindow(ctx, &j, fmt.Errorf("目标入口仍然可用:%w", stopErr), "ingress_open")
}
j.IngressClosed = true
}
j.State = "held"
if err = c.save(j); err != nil {
return Result{}, err
}
w := &Window{c: c, j: &j}
// work 里 panic 等同于失败:目标可能已被改过,必须走恢复分支,不能让
// 进程直接崩掉把写入者留在停止状态。
snapshotID, workErr := func() (id string, err error) {
defer func() {
if recover() != nil {
id, err = "", errors.New("维护窗口内的作业 panic")
}
}()
return work(ctx, w)
}()
if workErr != nil {
return c.closeWindow(ctx, &j, workErr, allowRestore(workErr), "")
}
if err = ctx.Err(); err != nil {
// 取消/超时发生在窗口内:内容状态未知,但写入者必须恢复(旧行为一致),
// 由上层把作业判失败;不把"没验证完"说成成功。
return c.closeWindow(ctx, &j, err, true, "")
}
if snapshotID != "" {
if !validDigest(snapshotID) {
return c.closeWindow(ctx, &j, errors.New("invalid candidate snapshot digest"), true, "")
}
}
// work 报成功但写入者已经不在停住状态 = 隔离被破坏,不能登记成功。
if err = c.stopped(ctx, j); err != nil {
return c.closeWindow(ctx, &j, err, true, "")
}
return c.closeWindow(ctx, &j, nil, true, snapshotID)
}
// abandonWindow:在"还没写目标"的阶段失败——内容没被改过,可以安全恢复并结案。
func (c *Coordinator) abandonWindow(ctx context.Context, j *journal, cause error, code string) (Result, error) {
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
defer cancel()
if restoreErr := c.restore(recoveryCtx, j); restoreErr != nil {
// 恢复不了就保持非终态:下一次恢复/接管还要处理它。
j.Code = code
_ = c.save(*j)
return j.Result, errors.Join(cause, restoreErr)
}
ended := time.Now().UTC()
j.State = "failed"
j.Code = code
j.FinishedAt = &ended
if err := c.save(*j); err != nil {
return j.Result, errors.Join(cause, err)
}
return j.Result, cause
}
// closeWindow:写完之后的收尾。allowRestore=false(半写且回滚失败)时
// 服务保持停止、journal 保持非终态,返回的是"目标仍在维护"这个结论。
func (c *Coordinator) closeWindow(ctx context.Context, j *journal, cause error, allowRestore bool, snapshotID string) (Result, error) {
if snapshotID != "" {
j.SnapshotID = snapshotID
}
if !allowRestore {
j.State = "held"
j.RestoreBlocked = true
if cause == nil {
cause = errors.New("目标内容状态未验证")
}
j.Code = "restore_blocked"
if err := c.save(*j); err != nil {
return j.Result, errors.Join(cause, err)
}
return j.Result, cause
}
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
defer cancel()
if restoreErr := c.restore(recoveryCtx, j); restoreErr != nil {
// 服务没能恢复 = 入口没回来 = 目标仍在维护,不许报成功。
j.State = "held"
j.Code = "restore_failed"
if saveErr := c.save(*j); saveErr != nil {
return j.Result, errors.Join(cause, restoreErr, saveErr)
}
return j.Result, errors.Join(cause, restoreErr)
}
if c.ingressReady != nil {
if readyErr := c.ingressReady(recoveryCtx); readyErr != nil {
j.State = "held"
j.Code = "ingress_unverified"
_ = c.save(*j)
return j.Result, errors.Join(cause, fmt.Errorf("目标入口没有恢复:%w", readyErr))
}
}
ended := time.Now().UTC()
j.FinishedAt = &ended
if cause != nil {
j.State = "failed"
if j.Code == "" {
j.Code = "work_failed"
}
} else {
j.State = "succeeded"
j.Code = ""
}
if err := c.save(*j); err != nil {
return j.Result, errors.Join(cause, err)
}
return j.Result, cause
}
// stopAll 只对"原本在跑"的服务发停止指令,每发一个都先落盘意图:
// 进程在停止过程中死掉时,磁盘上仍能看出"哪些服务被要求停过"。
func (c *Coordinator) stopAll(ctx context.Context, j *journal) error {
var failures error
for i := range j.Services {
if !j.Services[i].Running || j.Services[i].StopRequested {
continue
}
j.Services[i].StopRequested = true
if err := c.save(*j); err != nil {
return err
}
if err := c.control.Stop(ctx, j.Services[i].ID); err != nil {
failures = errors.Join(failures, err)
break
}
}
return failures
}
// verifyDrain 用控制器报告的退出码判断"在途请求是否排空":
// 0 = 服务自己收尾退出;其它(143/137)= 被切断,不许当成排空。
// 声明了实例级维护时,拿不到退出码就是没有证据——按未排空处理。
func (c *Coordinator) verifyDrain(ctx context.Context, j *journal) error {
needed := 0
for _, s := range j.Services {
if s.Running && s.StopRequested {
needed++
}
}
if needed == 0 {
return nil
}
items, err := inventory(ctx, c.control)
if err != nil {
return err
}
actual := map[string]Service{}
for _, s := range items {
actual[s.ID] = s
}
verified := 0
for i := range j.Services {
s := &j.Services[i]
if !s.Running || !s.StopRequested {
continue
}
now, ok := actual[s.ID]
if !ok || now.Running {
return errors.New("test writer still running")
}
if now.ExitCode == nil {
if c.level == "instance" {
return fmt.Errorf("无法证明 %s 的在途请求已排空(控制器没有退出码)", s.ID)
}
continue
}
s.ExitCode = now.ExitCode
if *now.ExitCode != 0 {
return fmt.Errorf("写入者 %s 未优雅退出(exit=%d):在途请求没有完成排空", s.ID, *now.ExitCode)
}
verified++
}
j.DrainVerified = verified == needed
return nil
}
func validDigest(id string) bool {
decoded, err := hex.DecodeString(id)
return err == nil && len(decoded) == 32 && strings.ToLower(id) == id
}
+253
View File
@@ -0,0 +1,253 @@
package libraryfreeze
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
)
// exitControl 模拟真实 Docker 控制器:停止时报告退出码(0 = 自己优雅退出)。
type exitControl struct {
*fixtureControl
exit map[string]int
}
func (c *exitControl) Inventory(ctx context.Context) ([]Service, error) {
items, err := c.fixtureControl.Inventory(ctx)
if err != nil {
return nil, err
}
for i := range items {
if code, ok := c.exit[items[i].ID]; ok && !items[i].Running {
value := code
items[i].ExitCode = &value
}
}
return items, nil
}
func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) {
t.Helper()
root := t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
c := &exitControl{fixtureControl: &fixtureControl{services: []Service{
{ID: "app", Running: true, Healthy: true}, {ID: "store", Running: true, Healthy: true}}}, exit: map[string]int{"app": 0, "store": 0}}
engine, err := Open(root, "target-instance", c)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { engine.Close() })
if err := engine.SetTarget("instance", "http://127.0.0.1:1/health", nil, nil); err != nil {
t.Fatal(err)
}
return root, c, engine
}
// 维护窗口必须真的挡住写入:work 只能在所有写入者确认停下、入口确认关闭之后才开始。
func TestWindowStopsWritersBeforeWorkAndRestoresAfter(t *testing.T) {
_, c, e := instanceFixture(t)
var sawStopped, sawClosed bool
e.ingressClosed = func(context.Context) error { sawClosed = true; return nil }
result, err := e.RunWindow(context.Background(), "job-1", func(context.Context) error { return nil },
func(ctx context.Context, w *Window) (string, error) {
for _, s := range c.services {
if s.Running {
t.Fatal("work started while a writer was still running")
}
}
if err := w.MarkContentRisk(ctx); err != nil {
t.Fatal(err)
}
sawStopped = true
return strings.Repeat("b", 64), nil
}, func(error) bool { return true })
if err != nil {
t.Fatal(err)
}
if !sawStopped || !sawClosed {
t.Fatalf("stopped=%v closed=%v", sawStopped, sawClosed)
}
if result.State != "succeeded" {
t.Fatalf("result=%+v", result)
}
j, err := e.load("job-1")
if err != nil {
t.Fatal(err)
}
if !j.DrainVerified || !j.IngressClosed || !j.ContentRisk || j.Level != "instance" {
t.Fatalf("journal evidence missing: %+v", j)
}
if strings.Join(c.calls, ",") != "stop:app,stop:store,start:app,start:store" {
t.Fatalf("calls=%v", c.calls)
}
}
// 没有优雅退出证据(退出码非 0)时不许开始写目标:在途操作可能被切断。
func TestWindowRefusesWorkWhenDrainUnverified(t *testing.T) {
_, c, e := instanceFixture(t)
c.exit["app"] = 143
work := 0
_, err := e.RunWindow(context.Background(), "job-drain", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || !strings.Contains(err.Error(), "排空") {
t.Fatalf("err=%v", err)
}
if work != 0 {
t.Fatal("work ran without drain evidence")
}
for _, s := range c.services {
if !s.Running {
t.Fatal("writers were not restored after drain failure")
}
}
}
// 入口停不下来(还在对外服务)就不是维护窗口。
func TestWindowRefusesWorkWhenIngressStillServing(t *testing.T) {
_, c, e := instanceFixture(t)
e.ingressClosed = func(context.Context) error { return errors.New("ingress still serving") }
work := 0
_, err := e.RunWindow(context.Background(), "job-ingress", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || work != 0 {
t.Fatalf("err=%v work=%d", err, work)
}
for _, s := range c.services {
if !s.Running {
t.Fatal("writers were not restored after ingress failure")
}
}
}
// 内容回滚失败:服务必须保持停止,且崩溃恢复不许自动把它们起来。
func TestWindowKeepsWritersStoppedWhenRestoreBlocked(t *testing.T) {
root, c, e := instanceFixture(t)
_, err := e.RunWindow(context.Background(), "job-half", func(context.Context) error { return nil },
func(ctx context.Context, w *Window) (string, error) {
if err := w.MarkContentRisk(ctx); err != nil {
t.Fatal(err)
}
return "", errors.New("目标写到一半,回滚也失败")
}, func(error) bool { return false })
if err == nil {
t.Fatal("blocked restore reported success")
}
for _, s := range c.services {
if s.Running {
t.Fatalf("writer %s was restarted on a half-written target", s.ID)
}
}
j, err := e.load("job-half")
if err != nil || !j.RestoreBlocked || !j.ContentRisk {
t.Fatalf("journal=%+v err=%v", j, err)
}
// 进程重启后:恢复器不能自作主张把半写目标对外开放。
if err = e.Recover(context.Background()); err == nil {
t.Fatal("recover reopened a target with unverified content")
}
for _, s := range c.services {
if s.Running {
t.Fatalf("recover restarted %s", s.ID)
}
}
// 原作业接管:不允许重新 stop(写入者本来就没起来),也不从零重来。
adopted := 0
_, err = e.RunWindow(context.Background(), "job-half", func(context.Context) error { t.Fatal("prepare ran on adoption"); return nil },
func(ctx context.Context, w *Window) (string, error) { adopted++; return "", nil }, func(error) bool { return true })
if err != nil || adopted != 1 {
t.Fatalf("adopt err=%v adopted=%d", err, adopted)
}
for _, s := range c.services {
if !s.Running {
t.Fatalf("writer %s was not restored after adoption", s.ID)
}
}
if _, statErr := os.Stat(root + "/job-half.json"); statErr != nil {
t.Fatal(statErr)
}
}
// 写到一半进程被杀:journal 必须记住"内容有风险",重启恢复不得放行。
func TestWindowCrashDuringWriteKeepsTargetHeld(t *testing.T) {
root, c, e := instanceFixture(t)
e.ingressClosed = func(context.Context) error { return nil }
_, err := e.RunWindow(context.Background(), "job-kill", func(context.Context) error { return nil },
func(ctx context.Context, w *Window) (string, error) {
if err := w.MarkContentRisk(ctx); err != nil {
t.Fatal(err)
}
return "", errors.New("worker 被 kill")
}, func(error) bool { return false })
if err == nil {
t.Fatal("kill scenario reported success")
}
j, err := e.load("job-kill")
if err != nil || !j.ContentRisk || j.State != "held" {
t.Fatalf("journal=%+v err=%v", j, err)
}
// 重新打开(模拟进程重启):磁盘上的证据必须足以阻止自动开放。
reopened, err := Open(root, "target-instance", c)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
if err = reopened.Recover(context.Background()); err == nil {
t.Fatal("recovery released a target with unverified content")
}
if len(c.calls) != 2 {
t.Fatalf("calls=%v", c.calls)
}
}
// 停止途中崩溃:journal 处于 stopping(内容还没风险),接管方可以继续停,
// 但不需要"重新准备",也不许把没停完的服务当成已停。
func TestWindowAdoptsStoppingJournal(t *testing.T) {
_, c, e := instanceFixture(t)
// 手工构造一个"停到一半"的现场:app 已停、store 还在跑。
e.control.(*exitControl).services[0].Running = false
j := journal{Version: 1, Source: "target-instance", Job: "job-mid", Level: "instance",
Ingress: "http://127.0.0.1:1/health", Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
j.Services = []serviceRecord{
{Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true},
{Service: Service{ID: "store", Running: true, Healthy: true}},
}
if err := e.save(j); err != nil {
t.Fatal(err)
}
_, err := e.RunWindow(context.Background(), "job-mid", func(context.Context) error { t.Fatal("prepare replayed"); return nil },
func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true })
if err != nil {
t.Fatal(err)
}
if strings.Join(c.calls, ",") != "stop:store,start:app,start:store" {
t.Fatalf("calls=%v", c.calls)
}
}
// 只锁库的层级不许带入口:那会把"没挡入口"说成"挡了"。
func TestLibraryLevelRejectsIngressClaim(t *testing.T) {
root := t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
c := &fixtureControl{services: []Service{{ID: "app", Running: true, Healthy: true}}}
e, err := Open(root, "target-instance", c)
if err != nil {
t.Fatal(err)
}
defer e.Close()
if err = e.SetTarget("library", "http://127.0.0.1:1/health", nil, nil); err == nil {
t.Fatal("library level accepted an ingress claim")
}
if err = e.SetTarget("instance", "", nil, nil); err == nil {
t.Fatal("instance level accepted an empty ingress")
}
if err = e.SetTarget("", "", nil, nil); err == nil {
t.Fatal("empty level accepted")
}
}
+252
View File
@@ -0,0 +1,252 @@
// Package librarymaintenance 把"目标维护窗口"落成一个可验证的实例级动作:
// 关闭目标入口 → 停下全部声明的写入者 → 验证在途排空 → 交给发布链路写目标
// → 恢复原本在跑的服务与入口。
//
// 它不产生授权、不读生产凭据,也不把"只锁了治理库 scope"当成系统维护。
package librarymaintenance
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path/filepath"
"sort"
"strings"
"time"
"knowledge-governance/internal/libraryfreeze"
)
// LevelInstance 表示写入者被共享给整个实例:停它影响所有知识库,
// 因此窗口必须按"实例级维护"对外标注,而不是"某个库在维护"。
const LevelInstance = "instance"
// Target 是目标的精确身份:实例、入口、写入者、要写的库/索引/桶。
// Fingerprint 覆盖全部字段,用来发现"配置换了但作业还按旧目标跑"。
type Target struct {
Instance string `json:"instance"`
Level string `json:"level"`
Project string `json:"project,omitempty"`
Writers []string `json:"writers,omitempty"`
Ingress string `json:"ingress,omitempty"`
Database string `json:"database"`
Index string `json:"index"`
Bucket string `json:"bucket"`
Fingerprint string `json:"fingerprint,omitempty"`
}
// Config 来自部署注入的配置。任何一项缺失都不许"降级成库级维护"蒙过去。
type Config struct {
Instance string
Project string
Writers []string
NonWriters map[string]string
DockerSocket string
IngressURL string
Database string
Index string
Bucket string
Timeout time.Duration
}
type Manager struct {
target Target
source string
docker string
writers []string
nonWriters map[string]string
timeout time.Duration
client *http.Client
}
func New(cfg Config) (*Manager, error) {
instance := strings.TrimSpace(cfg.Instance)
project := strings.TrimSpace(cfg.Project)
ingress := strings.TrimSpace(cfg.IngressURL)
if instance == "" || project == "" {
return nil, errors.New("实例级维护配置缺少 LIBRARY_TARGET_INSTANCE/PROJECT")
}
if len(cfg.Writers) == 0 {
return nil, errors.New("实例级维护配置缺少写入者声明 LIBRARY_TARGET_WRITERS")
}
if !filepath.IsAbs(cfg.DockerSocket) {
return nil, errors.New("Docker socket 必须是绝对路径")
}
parsed, err := url.Parse(ingress)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.Path == "" || parsed.User != nil {
return nil, errors.New("目标入口地址无效(LIBRARY_TARGET_INGRESS_URL)")
}
if cfg.Database == "" || cfg.Index == "" || cfg.Bucket == "" {
return nil, errors.New("目标身份不完整:需要数据库/索引/存储桶")
}
if cfg.Timeout <= 0 {
cfg.Timeout = 2 * time.Minute
}
writers := append([]string(nil), cfg.Writers...)
sort.Strings(writers)
target := Target{Instance: instance, Level: LevelInstance, Project: project, Writers: writers,
Ingress: ingress, Database: cfg.Database, Index: cfg.Index, Bucket: cfg.Bucket}
fingerprint, err := fingerprintOf(target)
if err != nil {
return nil, err
}
target.Fingerprint = fingerprint
return &Manager{target: target, source: project, docker: cfg.DockerSocket,
writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout,
client: &http.Client{Timeout: 5 * time.Second}}, nil
}
func fingerprintOf(t Target) (string, error) {
canonical := struct {
Instance string `json:"instance"`
Level string `json:"level"`
Project string `json:"project"`
Writers []string `json:"writers"`
Ingress string `json:"ingress"`
Database string `json:"database"`
Index string `json:"index"`
Bucket string `json:"bucket"`
}{t.Instance, t.Level, t.Project, t.Writers, t.Ingress, t.Database, t.Index, t.Bucket}
raw, err := json.Marshal(canonical)
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
func (m *Manager) Target() Target { return m.target }
// Description 是写进维护记录的一句话身份,含指纹,便于事后核对"当时动的是谁"。
func (m *Manager) Description() string {
t := m.target
return fmt.Sprintf("level=%s instance=%s project=%s writers=%s ingress=%s db=%s index=%s bucket=%s fingerprint=%s",
t.Level, t.Instance, t.Project, strings.Join(t.Writers, "+"), t.Ingress, t.Database, t.Index, t.Bucket, t.Fingerprint)
}
// Open 返回绑定到该目标的协调器与控制器。调用方负责 Close。
func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.ProjectController, error) {
control, err := libraryfreeze.NewProjectController(m.docker, m.source, m.writers, m.nonWriters, m.timeout)
if err != nil {
return nil, nil, err
}
coordinator, err := libraryfreeze.Open(root, m.source, control)
if err != nil {
control.Close()
return nil, nil, err
}
if err := coordinator.SetTarget(LevelInstance, m.target.Ingress, m.ingressClosed, m.ingressReady); err != nil {
coordinator.Close()
control.Close()
return nil, nil, err
}
return coordinator, control, nil
}
// OriginalState 是"动之前"的原服务状态,落在维护记录里:
// 只按它恢复,绝不顺手把原本没跑的服务拉起来。
func (m *Manager) OriginalState(ctx context.Context) (string, error) {
control, err := libraryfreeze.NewProjectController(m.docker, m.source, m.writers, m.nonWriters, m.timeout)
if err != nil {
return "", err
}
defer control.Close()
items, err := control.Inventory(ctx)
if err != nil {
return "", err
}
parts := make([]string, 0, len(items))
for _, item := range items {
parts = append(parts, fmt.Sprintf("%s=%s running=%t healthy=%t", m.serviceOf(item.ID), item.ID, item.Running, item.Healthy))
}
sort.Strings(parts)
return strings.Join(parts, "; "), nil
}
func (m *Manager) serviceOf(id string) string {
return "container:" + id
}
// VerifyRunning 确认目标入口真的在服务(部署与恢复后都要看,不看容器状态)。
func (m *Manager) VerifyRunning(ctx context.Context) error {
return m.ingressReady(ctx)
}
// ingressClosed:入口关掉的证据是"连不上",不是"返回错误页"——
// 502/503 说明还有一层代理在对外服务,那不是维护窗口。
func (m *Manager) ingressClosed(ctx context.Context) error {
deadline := time.Now().Add(m.timeout)
var last error
for {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.target.Ingress, nil)
if err != nil {
return err
}
resp, err := m.client.Do(req)
if err != nil {
return nil
}
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
resp.Body.Close()
last = fmt.Errorf("入口仍返回 HTTP %d", resp.StatusCode)
if time.Now().After(deadline) {
return last
}
select {
case <-ctx.Done():
return last
case <-time.After(200 * time.Millisecond):
}
}
}
// ingressReady:入口回来 = 真的能返回 200,而不是"容器起来了"。
func (m *Manager) ingressReady(ctx context.Context) error {
deadline := time.Now().Add(m.timeout)
var last error
for {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.target.Ingress, nil)
if err != nil {
return err
}
resp, err := m.client.Do(req)
if err == nil {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return nil
}
last = fmt.Errorf("入口返回 HTTP %d:%s", resp.StatusCode, strings.TrimSpace(string(body)))
} else {
last = err
}
if time.Now().After(deadline) {
return fmt.Errorf("目标入口没有恢复:%w", last)
}
select {
case <-ctx.Done():
return last
case <-time.After(500 * time.Millisecond):
}
}
}
// DatabaseIdentity 把连接串收敛成"host:port/db"这种可以写进维护记录的身份:
// 维护记录里不落库凭据,但必须能说清动的是哪个库。
func DatabaseIdentity(raw string) (string, error) {
parsed, err := url.Parse(strings.TrimSpace(raw))
if err != nil || parsed.Host == "" {
return "", errors.New("目标数据库连接串无效")
}
database := strings.TrimPrefix(parsed.Path, "/")
if database == "" {
return "", errors.New("目标数据库连接串缺少库名")
}
return parsed.Host + "/" + database, nil
}
+134
View File
@@ -0,0 +1,134 @@
package librarymaintenance
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func baseConfig() Config {
return Config{
Instance: "kgmaint-weknora", Project: "kgmaint-weknora",
Writers: []string{"app"},
NonWriters: map[string]string{"postgres": "database store; not an initiator"},
DockerSocket: "/var/run/docker.sock",
IngressURL: "http://127.0.0.1:18080/health",
Database: "127.0.0.1:25432/kgmaint_target",
Index: "kgmaint_1028a_chunks", Bucket: "kgmaint-1028a",
}
}
// 只锁治理库 scope 的"维护"不许被配置成实例级:那会冒充系统维护。
func TestNewRequiresExactInstanceIdentity(t *testing.T) {
for name, mutate := range map[string]func(*Config){
"no-instance": func(c *Config) { c.Instance = "" },
"no-project": func(c *Config) { c.Project = "" },
"no-writers": func(c *Config) { c.Writers = nil },
"no-ingress": func(c *Config) { c.IngressURL = "" },
"bad-ingress": func(c *Config) { c.IngressURL = "ftp://host" },
"no-index": func(c *Config) { c.Index = "" },
"no-bucket": func(c *Config) { c.Bucket = "" },
} {
t.Run(name, func(t *testing.T) {
cfg := baseConfig()
mutate(&cfg)
if _, err := New(cfg); err == nil {
t.Fatal("incomplete target accepted")
}
})
}
}
// 指纹必须随目标身份变化:换了索引/桶/入口就是另一个目标。
func TestFingerprintTracksTargetIdentity(t *testing.T) {
base, err := New(baseConfig())
if err != nil {
t.Fatal(err)
}
if !strings.Contains(base.Description(), base.Target().Fingerprint) {
t.Fatal("description omits fingerprint")
}
for name, mutate := range map[string]func(*Config){
"index": func(c *Config) { c.Index = "other_chunks" },
"bucket": func(c *Config) { c.Bucket = "other-bucket" },
"ingress": func(c *Config) { c.IngressURL = "http://127.0.0.1:18081/health" },
"writers": func(c *Config) { c.Writers = []string{"app", "worker"} },
} {
t.Run(name, func(t *testing.T) {
cfg := baseConfig()
mutate(&cfg)
other, err := New(cfg)
if err != nil {
t.Fatal(err)
}
if other.Target().Fingerprint == base.Target().Fingerprint {
t.Fatalf("fingerprint did not change for %s", name)
}
})
}
}
// 入口"关了"必须是连不上;代理还返回 502 不算关。
func TestIngressClosedRequiresConnectionRefused(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusBadGateway)
}))
cfg := baseConfig()
cfg.IngressURL = server.URL + "/health"
manager, err := New(cfg)
if err != nil {
t.Fatal(err)
}
manager.timeout = 400 * time.Millisecond
if err = manager.ingressClosed(context.Background()); err == nil {
t.Fatal("a still-serving proxy was treated as a closed ingress")
}
server.Close()
manager.client.CloseIdleConnections()
if err = manager.ingressClosed(context.Background()); err != nil {
t.Fatalf("closed ingress rejected: %v", err)
}
}
// 入口"回来了"必须是 200;容器 running 不算。
func TestIngressReadyRequiresHealthyResponse(t *testing.T) {
status := http.StatusServiceUnavailable
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(status)
}))
defer server.Close()
cfg := baseConfig()
cfg.IngressURL = server.URL + "/health"
manager, err := New(cfg)
if err != nil {
t.Fatal(err)
}
manager.timeout = 300 * time.Millisecond
if err = manager.VerifyRunning(context.Background()); err == nil {
t.Fatal("unhealthy ingress reported ready")
}
status = http.StatusOK
if err = manager.VerifyRunning(context.Background()); err != nil {
t.Fatalf("healthy ingress rejected: %v", err)
}
}
// 维护记录里的库身份不含凭据。
func TestDatabaseIdentityDropsCredentials(t *testing.T) {
identity, err := DatabaseIdentity("postgres://user:secret@127.0.0.1:25432/kgmaint_target?sslmode=disable")
if err != nil {
t.Fatal(err)
}
if identity != "127.0.0.1:25432/kgmaint_target" {
t.Fatalf("identity=%q", identity)
}
if strings.Contains(identity, "secret") {
t.Fatal("credentials leaked into identity")
}
if _, err = DatabaseIdentity("not-a-dsn"); err == nil {
t.Fatal("invalid DSN accepted")
}
}
@@ -2,6 +2,7 @@ package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
@@ -10,10 +11,18 @@ import (
)
// LibraryMaintenance 是目标维护窗口的持有记录:一个 scope 只能有一个持有者。
//
// Level 说明这次维护到底动了什么:
// - "instance":入口与写入者被真的关掉(实例级),所有库都受影响;
// - "library":只锁了治理库的一个 scope,目标入口/后台写入没被挡,不能当成系统维护。
type LibraryMaintenance struct {
Scope string `json:"scope"`
OwnerJobID string `json:"owner_job_id"`
TargetIdentity string `json:"target_identity"`
Level string `json:"level"`
TargetFingerprint string `json:"target_fingerprint,omitempty"`
Services []string `json:"services,omitempty"`
IngressState string `json:"ingress_state"`
State string `json:"state"`
OriginalServiceState string `json:"original_service_state"`
Note string `json:"note"`
@@ -25,22 +34,48 @@ type LibraryMaintenance struct {
// ErrMaintenanceHeld 表示目标维护窗口被别的作业占着:任何新的写入都必须拒绝。
var ErrMaintenanceHeld = errors.New("目标正在维护中,被其他作业占用")
const maintenanceColumns = `scope, owner_job_id, target_identity, level, target_fingerprint, services,
ingress_state, state, original_service_state, note, acquired_at, updated_at, released_at`
// AcquireLibraryMaintenance 原子地拿到维护权:
// 同一作业可以重入(接管后继续),别的作业一律拒绝——这不是"尽力而为"的锁。
func (s *Store) AcquireLibraryMaintenance(ctx context.Context, item LibraryMaintenance, now time.Time) (LibraryMaintenance, error) {
level := item.Level
if level == "" {
level = "library"
}
ingress := item.IngressState
if ingress == "" {
ingress = "unknown"
}
services := item.Services
if services == nil {
services = []string{}
}
raw, err := json.Marshal(services)
if err != nil {
return LibraryMaintenance{}, fmt.Errorf("acquire library maintenance: %w", err)
}
row := s.db.QueryRow(ctx, `
INSERT INTO library_maintenance (scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$7)
INSERT INTO library_maintenance (scope, owner_job_id, target_identity, level, target_fingerprint, services,
ingress_state, state, original_service_state, note, acquired_at, updated_at)
VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$11)
ON CONFLICT (scope) DO UPDATE SET
owner_job_id = EXCLUDED.owner_job_id,
target_identity = EXCLUDED.target_identity,
level = EXCLUDED.level,
target_fingerprint = EXCLUDED.target_fingerprint,
services = EXCLUDED.services,
ingress_state = EXCLUDED.ingress_state,
state = EXCLUDED.state,
original_service_state = EXCLUDED.original_service_state,
note = EXCLUDED.note,
updated_at = EXCLUDED.updated_at,
released_at = NULL
WHERE library_maintenance.owner_job_id = EXCLUDED.owner_job_id
RETURNING scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at, released_at`,
item.Scope, item.OwnerJobID, item.TargetIdentity, item.State, item.OriginalServiceState, item.Note, now.UTC())
RETURNING `+maintenanceColumns,
item.Scope, item.OwnerJobID, item.TargetIdentity, level, item.TargetFingerprint, string(raw),
ingress, item.State, item.OriginalServiceState, item.Note, now.UTC())
acquired, err := scanMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return LibraryMaintenance{}, ErrMaintenanceHeld
@@ -64,6 +99,23 @@ WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID,
return nil
}
// SetLibraryMaintenanceIngress 记录入口的真实状态(closed/open),
// 页面据此区分"入口关了"和"只是容器停了"。
func (s *Store) SetLibraryMaintenanceIngress(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error {
if state != "closed" && state != "open" {
return errors.New("invalid ingress state")
}
tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET ingress_state=$3, note=$4, updated_at=$5
WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, state, note, now.UTC())
if err != nil {
return fmt.Errorf("update library maintenance ingress: %w", err)
}
if tag.RowsAffected() == 0 {
return errors.New("library maintenance not held by this job")
}
return nil
}
// ReleaseLibraryMaintenance 只在完整验证通过后调用;失败路径必须保持窗口。
func (s *Store) ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET state='released', released_at=$3, updated_at=$3
@@ -79,8 +131,7 @@ WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID,
// LibraryMaintenanceOf 读当前窗口状态(页面与诊断用)。
func (s *Store) LibraryMaintenanceOf(ctx context.Context, scope string) (*LibraryMaintenance, error) {
row := s.db.QueryRow(ctx, `SELECT scope, owner_job_id, target_identity, state, original_service_state, note,
acquired_at, updated_at, released_at FROM library_maintenance WHERE scope=$1`, scope)
row := s.db.QueryRow(ctx, `SELECT `+maintenanceColumns+` FROM library_maintenance WHERE scope=$1`, scope)
item, err := scanMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
@@ -93,7 +144,17 @@ acquired_at, updated_at, released_at FROM library_maintenance WHERE scope=$1`, s
func scanMaintenance(row pgx.Row) (LibraryMaintenance, error) {
var item LibraryMaintenance
err := row.Scan(&item.Scope, &item.OwnerJobID, &item.TargetIdentity, &item.State,
&item.OriginalServiceState, &item.Note, &item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt)
return item, err
var raw []byte
err := row.Scan(&item.Scope, &item.OwnerJobID, &item.TargetIdentity, &item.Level, &item.TargetFingerprint,
&raw, &item.IngressState, &item.State, &item.OriginalServiceState, &item.Note,
&item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt)
if err != nil {
return LibraryMaintenance{}, err
}
if len(raw) > 0 {
if err := json.Unmarshal(raw, &item.Services); err != nil {
return LibraryMaintenance{}, fmt.Errorf("maintenance services unreadable: %w", err)
}
}
return item, nil
}
@@ -0,0 +1,5 @@
ALTER TABLE library_maintenance
DROP COLUMN IF EXISTS ingress_state,
DROP COLUMN IF EXISTS services,
DROP COLUMN IF EXISTS target_fingerprint,
DROP COLUMN IF EXISTS level;
@@ -0,0 +1,9 @@
-- 维护记录要能区分"实例级维护"(入口与写入者被真的关掉)与"仅治理库互斥"。
-- 也要能事后核对:动的是哪个实例、哪几个容器、入口当时是不是真的关了。
ALTER TABLE library_maintenance
ADD COLUMN level VARCHAR(16) NOT NULL DEFAULT 'library'
CHECK (level IN ('library', 'instance')),
ADD COLUMN target_fingerprint VARCHAR(64) NOT NULL DEFAULT '',
ADD COLUMN services JSONB NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN ingress_state VARCHAR(16) NOT NULL DEFAULT 'unknown'
CHECK (ingress_state IN ('unknown', 'closed', 'open'));
+14 -9
View File
@@ -11,25 +11,30 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 70 {
t.Fatalf("migration count = %d, want 70", len(migrations))
if len(migrations) != 71 {
t.Fatalf("migration count = %d, want 71", len(migrations))
}
if tail := migrations[len(migrations)-2]; tail.Version != 69 ||
// 71 让维护记录能区分实例级维护与仅治理库互斥。
if level := migrations[len(migrations)-1]; level.Version != 71 ||
!strings.Contains(level.UpSQL, "level") || !strings.Contains(level.UpSQL, "ingress_state") {
t.Fatalf("migration 71 = %#v", level)
}
if maint := migrations[len(migrations)-2]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
t.Fatalf("migration 70 = %#v", maint)
}
if tail := migrations[len(migrations)-3]; tail.Version != 69 ||
!strings.Contains(tail.UpSQL, "library_jobs_write_target") {
t.Fatalf("migration 69 = %#v", tail)
}
if maint := migrations[len(migrations)-1]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
t.Fatalf("migration 70 = %#v", maint)
}
if restore := migrations[len(migrations)-3]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
if restore := migrations[len(migrations)-4]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
t.Fatalf("migration 68 = %#v", restore)
}
if last := migrations[len(migrations)-4]; last.Version != 67 ||
if last := migrations[len(migrations)-5]; last.Version != 67 ||
!strings.Contains(last.UpSQL, "library_release_authorizations") ||
!strings.Contains(last.UpSQL, "params_digest") {
t.Fatalf("migration 67 = %#v", last)
}
if flow := migrations[len(migrations)-5]; flow.Version != 66 ||
if flow := migrations[len(migrations)-6]; flow.Version != 66 ||
!strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") ||
!strings.Contains(flow.UpSQL, "library_jobs_active") {
t.Fatalf("migration 66 = %#v", flow)