姚章浩
517d5f862d
目标非空又没有生效版本时，首次发布以前只能停在"权威版本未知"。现在有了 真实入口：管理员发起 baseline 作业，worker 在维护隔离下把目标整库捕获成 归档快照，再用同一套 Verify 证明目标与快照逐类一致，最后登记 production_effective（证据写明 origin=target_baseline、快照 ID、逐表行数、 目标索引映射）。不写目标内容、不清空目标、不手写版本。 同时修两处会误导人的判定： - productionScopeEmpty 原来只数 knowledges，"文档删了但分块/Wiki/资源还在" 的目标会被当成空库；改用适配层同一份表清单与范围过滤逐表计数。 - 概览里的"测试封存候选"原来取该 scope 最新一份归档，而归档里还有生产 预备份与目标基线快照；改为只认版本登记里 kind=test_sealed 的那一份。 迁移 075 放开 kind='baseline' 并把它并入 sync/restore 的写互斥索引； worker 重启时用同一作业 id 接管未完成的基线窗口。维护接口补 capability， 页面才能区分"未运行维护"与"维护能力没接线"。 验证：go build ./... / go vet ./... 通过；新增用例覆盖管理员校验、已有生效 版本拒绝、目标读不到时零作业零版本、候选来自登记、缺项文案指向基线入口、 capability 逐项缺失；反向自检（把候选改回按最新归档取、把文案改回旧句、 去掉管理员判断）三项均按预期变红后复原。
585 lines
29 KiB
Go
585 lines
29 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryfreeze"
|
||
"knowledge-governance/internal/librarymaintenance"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
"knowledge-governance/internal/pgdsn"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
)
|
||
|
||
// 目标内容在窗口内的几种"已知状态"，决定目标能否被重新开放：
|
||
// - contentUntouched：这次窗口还没写过目标（可以安全开放）
|
||
// - contentSucceeded / contentRolledBack：内容已验证，或已回滚并验证（可以开放）
|
||
// - contentHalfWritten / contentRollbackFails：内容可能半写（必须保持维护）
|
||
const (
|
||
contentUntouched = ""
|
||
contentSucceeded = "succeeded"
|
||
contentRolledBack = "rolled_back"
|
||
contentHalfWritten = "half_written"
|
||
contentRollbackFails = "rollback_failed"
|
||
)
|
||
|
||
// contentTrackedTarget 只做一件事：在第一次写目标时把"内容有风险"标出来。
|
||
// 这样"授权过期/进程被杀/校验失败"这些早退路径不会把半写目标当成没动过。
|
||
type contentTrackedTarget struct {
|
||
inner libraryrelease.Target
|
||
onApply func()
|
||
}
|
||
|
||
func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) {
|
||
return t.inner.Current(ctx)
|
||
}
|
||
|
||
func (t *contentTrackedTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error {
|
||
t.onApply()
|
||
return t.inner.Apply(ctx, stage, snapshotID)
|
||
}
|
||
|
||
func (t *contentTrackedTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error {
|
||
return t.inner.Verify(ctx, expect)
|
||
}
|
||
|
||
func (t *contentTrackedTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error {
|
||
return t.inner.Restore(ctx, backup)
|
||
}
|
||
|
||
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
|
||
type libraryReleasePayload struct {
|
||
LibraryJobID string `json:"library_job_id"`
|
||
Scope string `json:"scope"`
|
||
TargetSnapshotID string `json:"target_snapshot_id"`
|
||
// FinalizeOnly：只做收尾（校验 + 登记生效版本），不重写目标。
|
||
// 用于"写入已完成但版本登记失败"的续作，避免把已生效的生产再写一遍。
|
||
FinalizeOnly bool `json:"finalize_only,omitempty"`
|
||
}
|
||
|
||
// openLibraryMaintenance 按部署注入的配置装配实例级维护目标。
|
||
// 未接线时返回明确原因：只锁治理库 scope 不算维护，宁可拒绝写也不冒充。
|
||
func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) {
|
||
database, err := librarymaintenance.DatabaseIdentity(config.LibraryProductionDatabaseURL)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return librarymaintenance.New(librarymaintenance.Config{
|
||
Instance: config.LibraryTargetInstance,
|
||
Project: config.LibraryTargetProject,
|
||
Writers: config.LibraryTargetWriters,
|
||
NonWriters: config.LibraryTargetNonWriters,
|
||
DockerSocket: config.LibraryTargetDockerSocket,
|
||
IngressURL: config.LibraryTargetIngressURL,
|
||
IngressService: config.LibraryTargetIngressService,
|
||
IngressDrainFile: config.LibraryTargetIngressDrainFile,
|
||
Database: database,
|
||
Index: config.LibraryProductionIndex,
|
||
Bucket: config.LibraryProductionS3Bucket,
|
||
// 后台静默检查的运行时依赖：HTTP 排空不等于后台任务排空。
|
||
DatabaseURL: config.LibraryProductionDatabaseURL,
|
||
RedisAddr: config.LibraryTargetRedisAddr,
|
||
RedisPassword: config.LibraryTargetRedisPassword,
|
||
RedisDB: config.LibraryTargetRedisDB,
|
||
})
|
||
}
|
||
|
||
// runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 →
|
||
// 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。
|
||
// 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。
|
||
func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config appconfig.Config,
|
||
job jobs.Job, kind governance.LibraryJobKind) error {
|
||
var payload libraryReleasePayload
|
||
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" {
|
||
return fmt.Errorf("invalid library %s payload", kind)
|
||
}
|
||
now := time.Now().UTC()
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
|
||
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope}
|
||
// 四类进度的真实总量来自快照清单（manifest）：读不到就保持 NULL，不写 0。
|
||
if archive, openErr := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024); openErr == nil {
|
||
if manifest, loadErr := archive.Load(payload.TargetSnapshotID); loadErr == nil {
|
||
recorder.totals = stageSizesFrom(manifest)
|
||
}
|
||
archive.Close()
|
||
}
|
||
|
||
// 崩溃接管：作业里已经持久化过预备份快照，就直接复用它继续，
|
||
// 绝不重新备份——重新备份会把"半写状态"当成新基线，覆盖真正可回滚的那份。
|
||
previous, err := store.LibraryJob(ctx, payload.LibraryJobID)
|
||
resumed := err == nil && previous.SnapshotID != ""
|
||
if resumed {
|
||
versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50)
|
||
switch classifyTakeover(previous, versions, versionErr) {
|
||
case takeoverUnknown:
|
||
// 版本状态读不到、又没有阶段证据：不能当成"没有收尾证据"继续写目标。
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"接管时读不到该目标的版本状态，无法判断是只差收尾还是要重做；未对目标执行任何写入，稍后重试", "prebackup")
|
||
case takeoverSuperseded:
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"本快照的生效版本已被别的快照顶替，拒绝用旧快照覆盖当前生产版本", "prebackup")
|
||
case takeoverPendingFinal:
|
||
// 上次可能死在"锁已放、终态没写"之间：只复核与登记，不重写生产。
|
||
payload.FinalizeOnly = true
|
||
}
|
||
}
|
||
if resumed {
|
||
recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID)
|
||
}
|
||
|
||
gateReasons := make([]string, 0, 3)
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
gateReasons = append(gateReasons, "生产写入闸口关闭")
|
||
}
|
||
if !config.ProductionBackupRestoreVerified {
|
||
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
|
||
}
|
||
if len(gateReasons) > 0 {
|
||
message := "生产" + kindLabel(kind) + "被闸口拒绝：" + strings.Join(gateReasons, "；")
|
||
recorder.Stage(ctx, "prebackup", "failed", message)
|
||
// 接管场景下生产可能已经是半写状态：这时"无需恢复"是错的，
|
||
// 必须把既有预备份作为恢复基线留住，并保持目标被占用直到处理完。
|
||
recovery := "未对生产执行任何写入，无需恢复"
|
||
if resumed {
|
||
recovery = "生产可能处于半写状态，必须先按预备份 " + previous.SnapshotID + " 恢复后再重试"
|
||
}
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
|
||
recovery, previous.SnapshotID, "", time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
|
||
digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID})
|
||
if err != nil {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "无法计算请求摘要", "prebackup")
|
||
}
|
||
grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now)
|
||
if err != nil || !found {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "没有与目标快照匹配的有效发布授权", "prebackup")
|
||
}
|
||
// 生产索引映射基线：优先当前生效版本，其次任何一次成功预备份（它也记录了生产实际映射）。
|
||
// 只认"生效版本"会在恢复把版本 supersede 掉之后误判"索引被改动"。
|
||
knownMapping := ""
|
||
if versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50); versionErr == nil {
|
||
for _, item := range versions {
|
||
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
|
||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||
knownMapping = raw
|
||
break
|
||
}
|
||
}
|
||
}
|
||
if knownMapping == "" {
|
||
for _, item := range versions {
|
||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||
knownMapping = raw
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping)
|
||
if wireErr == nil {
|
||
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
|
||
recorder.mappingDigest = mapping
|
||
}
|
||
}
|
||
if wireErr != nil {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup")
|
||
}
|
||
defer target.Close()
|
||
|
||
// 维护目标必须在真正动目标之前确定：缺配置就拒绝写，不用"库级锁"顶替。
|
||
manager, managerErr := openLibraryMaintenance(config)
|
||
if managerErr != nil {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"目标维护未接线，拒绝在挡不住入口/后台写入的情况下写目标："+managerErr.Error(), "prebackup")
|
||
}
|
||
defer manager.Close()
|
||
// 拓扑必须先过关：写入者发布了端口 = 存在绕闸直连，窗口关的只是半扇门。
|
||
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"目标拓扑不满足唯一入口约束，拒绝开维护窗口："+topoErr.Error(), "prebackup")
|
||
}
|
||
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
|
||
pgdsn.Endpoint(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
|
||
originalState, stateErr := manager.OriginalState(ctx)
|
||
if stateErr != nil {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"读不到目标写入者的原运行状态，不能开维护窗口："+stateErr.Error(), "prebackup")
|
||
}
|
||
identity := manager.Target()
|
||
// 实例级占用：internal/external 动的是同一批容器，必须按实例互斥。
|
||
instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
|
||
Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope,
|
||
TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance,
|
||
Services: identity.Writers, IngressState: "unknown", State: "acquired",
|
||
OriginalServiceState: originalState,
|
||
Note: "实例级维护：关闭目标入口并停止全部写入者，验证或回滚完成才开放",
|
||
}, now)
|
||
if instanceErr != nil {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||
"目标实例正在维护中（跨对内/对外互斥）："+instanceErr.Error(), "prebackup")
|
||
}
|
||
_ = instanceLock
|
||
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
|
||
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID,
|
||
TargetIdentity: targetIdentity + " | " + manager.Description(),
|
||
Level: librarymaintenance.LevelInstance,
|
||
TargetFingerprint: identity.Fingerprint,
|
||
Services: identity.Writers,
|
||
IngressState: "unknown",
|
||
State: "acquired",
|
||
OriginalServiceState: originalState,
|
||
Note: "实例级维护：关闭目标入口并停止全部写入者，验证或回滚完成才开放",
|
||
}, now)
|
||
if acquireErr != nil {
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
|
||
"scope 占用申请失败，实例状态未确认："+acquireErr.Error(), now)
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中："+acquireErr.Error(), "prebackup")
|
||
}
|
||
_ = maintenance
|
||
|
||
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
|
||
if openErr != nil {
|
||
// 协调器都打不开：这一次没动过目标。接管场景除外——上一次尝试可能已经把
|
||
// 写入者停住并写了一半，那种情况下保持占用是唯一安全的选择。
|
||
if resumed {
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
|
||
"维护协调器不可用，接管目标的状态未确认："+openErr.Error(), time.Now().UTC())
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
|
||
"维护协调器不可用，接管目标的状态未确认："+openErr.Error(), time.Now().UTC())
|
||
} else {
|
||
_ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||
"未进入维护窗口（协调器不可用），目标未被停用", time.Now().UTC())
|
||
}
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "维护协调器不可用："+openErr.Error(), "prebackup")
|
||
}
|
||
defer control.Close()
|
||
defer coordinator.Close()
|
||
|
||
// 接管场景下目标可能已经被上一次尝试写过：在证明"已回滚或已验证"之前，
|
||
// 一律按半写对待，任何早退都不许重新开放目标。
|
||
contentState := contentUntouched
|
||
if resumed {
|
||
contentState = contentHalfWritten
|
||
}
|
||
pending := successBoundary{}
|
||
tracked := &contentTrackedTarget{inner: target, onApply: func() { contentState = contentHalfWritten }}
|
||
allowRestore := func(error) bool {
|
||
return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack
|
||
}
|
||
|
||
execute := func(ctx context.Context) error {
|
||
if payload.FinalizeOnly {
|
||
// 写入已完成、只是版本登记失败：只做校验 + 登记，绝不重写生产。
|
||
recorder.Stage(ctx, "finalize", "running", "续作收尾：先校验生产与目标快照一致")
|
||
if verifyErr := tracked.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil {
|
||
contentState = contentHalfWritten
|
||
message := "生产与目标快照不一致，不能登记为生效版本：" + verifyErr.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
|
||
"生产可能处于半写状态，需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
contentState = contentSucceeded
|
||
mappingDigest, _ := target.Search().MappingSHA256(ctx)
|
||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
|
||
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
|
||
SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker",
|
||
Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only",
|
||
"index_mapping_sha256": mappingDigest},
|
||
CreatedAt: time.Now().UTC(),
|
||
}); err != nil {
|
||
message := "收尾续作仍无法登记生效版本：" + err.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
|
||
"预备份 "+previous.SnapshotID+" 保留，可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本（仅收尾，未重写生产）")
|
||
// 成功要等维护锁释放完才暴露：这里只记"待收尾"。
|
||
pending = successBoundary{Ready: true, Backup: previous.SnapshotID,
|
||
Target: payload.TargetSnapshotID, FinishedAt: time.Now().UTC()}
|
||
return nil
|
||
}
|
||
|
||
var backupSource libraryrelease.BackupStore = backups
|
||
if resumed {
|
||
backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
|
||
SnapshotID: previous.SnapshotID}}
|
||
}
|
||
|
||
request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID,
|
||
ParamsDigest: digest, Now: now,
|
||
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
|
||
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
|
||
outcome := libraryrelease.Run(ctx, request, tracked, backupSource, recorder)
|
||
finishedAt := time.Now().UTC()
|
||
switch outcome.State {
|
||
case "succeeded":
|
||
contentState = contentSucceeded
|
||
mappingDigest := ""
|
||
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
|
||
mappingDigest = mapping
|
||
}
|
||
// 收尾阶段：先登记权威版本，登记成功才允许把作业标成功。
|
||
// 登记失败必须让作业失败（可续作），否则"版本记录"和"作业状态"会分叉。
|
||
recorder.Stage(ctx, "finalize", "running", "登记生产生效版本")
|
||
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
|
||
Scope: payload.Scope,
|
||
Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID,
|
||
SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy,
|
||
Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID,
|
||
"backup_bytes": outcome.Backup.Bytes, "action": string(kind),
|
||
"index_mapping_sha256": mappingDigest},
|
||
CreatedAt: finishedAt,
|
||
})
|
||
if versionErr != nil {
|
||
message := "生产写入已完成，但生效版本登记失败：" + versionErr.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
// 作业不能报成功：预备份仍在，修好数据库后可用 resume 续作收尾。
|
||
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
|
||
message, "预备份 "+outcome.Backup.SnapshotID+" 保留，数据库恢复后可用续作重试登记",
|
||
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
|
||
log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr)
|
||
}
|
||
return errors.New(message)
|
||
}
|
||
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本")
|
||
// 内容与版本都落定了，但还不能报成功：服务健康、入口开放、两把维护锁
|
||
// 释放完成才允许暴露 succeeded（见 finishLibrarySuccess）。
|
||
pending = successBoundary{Ready: true, Backup: outcome.Backup.SnapshotID,
|
||
Target: payload.TargetSnapshotID, FinishedAt: finishedAt}
|
||
return nil
|
||
case "rolled_back":
|
||
contentState = contentRolledBack
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||
return errors.New(outcome.Error)
|
||
case "rollback_failed":
|
||
contentState = contentRollbackFails
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||
return fmt.Errorf("%s（%s）", outcome.Error, outcome.Rollback)
|
||
default:
|
||
recovery := outcome.Recovery
|
||
if recovery == "" {
|
||
recovery = "未对生产执行任何写入，无需恢复"
|
||
}
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup",
|
||
outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt)
|
||
return errors.New(outcome.Error)
|
||
}
|
||
}
|
||
|
||
prepare := func(ctx context.Context) error {
|
||
// 停下写入者之前必须确认目标现在是健康的：拿一个已经不健康的目标开窗口，
|
||
// 只会把"本来坏的"记成"维护造成的"。
|
||
return manager.VerifyRunning(ctx)
|
||
}
|
||
windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) {
|
||
// 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘：没有预备份身份的
|
||
// 窗口之后既不能自动恢复，也不许被任何收尾路径开放。
|
||
recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error {
|
||
if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil {
|
||
return err
|
||
}
|
||
return w.MarkContentRisk(backupCtx)
|
||
}
|
||
if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed",
|
||
"目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil {
|
||
// 记录失败不影响隔离本身，但必须留痕，不能假装记上了。
|
||
log.Printf("library %s maintenance ingress record failed: %v", payload.LibraryJobID, ingressErr)
|
||
}
|
||
return "", execute(ctx)
|
||
}
|
||
window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore)
|
||
|
||
// 收尾：只有"内容已验证/已回滚"且服务与入口都恢复，才释放维护占用。
|
||
now2 := time.Now().UTC()
|
||
switch {
|
||
case window.State == "held":
|
||
note := "目标保持维护：内容状态未验证（半写或回滚失败），或服务/入口未能恢复"
|
||
if windowErr != nil {
|
||
note += "：" + windowErr.Error()
|
||
}
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
|
||
if windowErr == nil {
|
||
return errors.New(note)
|
||
}
|
||
return windowErr
|
||
case contentState == contentSucceeded:
|
||
if !pending.Ready {
|
||
// 内容状态说成功但没有收尾凭据（不可能两者都缺）：按未收尾处理，保持维护。
|
||
note := "内容状态为已验证，但缺少收尾凭据，目标保持维护"
|
||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
|
||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
|
||
return errors.New(note)
|
||
}
|
||
if finishErr := finishLibrarySuccess(ctx, store, manager, identity.Instance, payload.Scope,
|
||
payload.LibraryJobID, pending); finishErr != nil {
|
||
return finishErr
|
||
}
|
||
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
|
||
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, pending.Backup)
|
||
return windowErr
|
||
case contentState == contentRolledBack:
|
||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||
"本次失败已回滚到预备份并验证，目标已恢复原运行状态", now2); err != nil {
|
||
return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err)
|
||
}
|
||
return windowErr
|
||
default:
|
||
// 没有写过目标（闸口/授权/接线/健康检查早退）：目标没被停过，直接开放占用。
|
||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||
"未写目标，维护窗口未开启", now2); err != nil {
|
||
return fmt.Errorf("维护占用释放失败: %w", err)
|
||
}
|
||
return windowErr
|
||
}
|
||
}
|
||
|
||
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业：
|
||
// 已经持久化预备份的重新排队（同一个业务作业 id，阶段与备份都接着用），
|
||
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线，不能盲写。
|
||
func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Store, config appconfig.Config) (int, int, error) {
|
||
interrupted, err := store.InterruptedLibraryJobs(ctx)
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
resumed, failed := 0, 0
|
||
for _, job := range interrupted {
|
||
now := time.Now().UTC()
|
||
if job.SnapshotID == "" {
|
||
message := "worker 中断时还没有完成预备份，不能盲写目标"
|
||
_ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message,
|
||
"未对生产执行写入", "", "", now)
|
||
failed++
|
||
continue
|
||
}
|
||
// 原子抢占：多个 worker 同时接管也只有一个能把 running 改成 pending。
|
||
claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now)
|
||
if claimErr != nil {
|
||
log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr)
|
||
continue
|
||
}
|
||
if !claimed {
|
||
continue
|
||
}
|
||
queueType := librarySyncJobType
|
||
switch job.Kind {
|
||
case governance.LibraryJobRestore:
|
||
queueType = libraryRestoreJobType
|
||
case governance.LibraryJobBaseline:
|
||
// 基线作业被杀时窗口可能还停着目标：用同一个作业 id 接管那个窗口，
|
||
// 继续捕获/校验，而不是留下一个 held 目标等人工处理。
|
||
queueType = libraryBaselineJobType
|
||
}
|
||
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
|
||
"target_snapshot_id": job.TargetSnapshotID})
|
||
// 队列作业 ID 与幂等键都按作业固定：重复接管/重复点击只会得到同一个队列作业。
|
||
queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{
|
||
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload,
|
||
AvailableAt: now, MaxAttempts: 1,
|
||
IdempotencyKey: "library.resume.queue:" + job.ID,
|
||
}, now)
|
||
if err != nil || queueJob == nil {
|
||
// 队列作业 ID 是确定性的：并发接管时另一个 worker 已经建好了，
|
||
// 这里按"已存在"处理并继续接管，不去覆盖别人的队列行。
|
||
if err != nil && strings.Contains(err.Error(), "duplicate key") {
|
||
queueID := governance.QueueJobIDFor(job.ID)
|
||
if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil {
|
||
log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr)
|
||
continue
|
||
}
|
||
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil {
|
||
log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr)
|
||
} else {
|
||
resumed++
|
||
}
|
||
continue
|
||
}
|
||
log.Printf("library job %s resume enqueue failed: %v", job.ID, err)
|
||
continue
|
||
}
|
||
if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil {
|
||
log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr)
|
||
continue
|
||
}
|
||
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil {
|
||
log.Printf("library job %s resume attach failed: %v", job.ID, attachErr)
|
||
continue
|
||
}
|
||
resumed++
|
||
log.Printf("library job %s resumed after worker restart (backup %s)", job.ID, job.SnapshotID)
|
||
}
|
||
return resumed, failed, nil
|
||
}
|
||
|
||
// fixedBackupStore 在崩溃接管时返回已经存在的预备份，不再新建。
|
||
type fixedBackupStore struct{ ref libraryrelease.BackupRef }
|
||
|
||
func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
|
||
return s.ref, nil
|
||
}
|
||
|
||
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误：
|
||
// 上游据此在写目标前中止，保证"没有可恢复基线就不写"。
|
||
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
|
||
now := time.Now().UTC()
|
||
if r.onBackup != nil {
|
||
if err := r.onBackup(ctx, backup); err != nil {
|
||
return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err)
|
||
}
|
||
}
|
||
if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "",
|
||
backup.SnapshotID, "", now); err != nil {
|
||
return fmt.Errorf("持久化预备份引用失败: %w", err)
|
||
}
|
||
if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID),
|
||
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID,
|
||
SourceJobID: r.jobID, CreatedBy: "worker",
|
||
Evidence: map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes,
|
||
"job": r.jobID, "index_mapping_sha256": r.mappingDigest},
|
||
CreatedAt: now,
|
||
}); err != nil {
|
||
return fmt.Errorf("登记预备份版本失败: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func failLibraryJob(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder, jobID, message, stage string) error {
|
||
recorder.Stage(ctx, stage, "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message,
|
||
"未对生产执行任何写入，无需恢复", "", "", time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
|
||
func kindLabel(kind governance.LibraryJobKind) string {
|
||
if kind == governance.LibraryJobRestore {
|
||
return "恢复"
|
||
}
|
||
return "同步"
|
||
}
|
||
|
||
func digestKey(kind governance.LibraryJobKind) string {
|
||
if kind == governance.LibraryJobRestore {
|
||
return "restore_snapshot_id"
|
||
}
|
||
return "target_snapshot_id"
|
||
}
|