package main import ( "context" "encoding/json" "errors" "fmt" "log" "strings" "time" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryfreeze" "knowledge-governance/internal/librarymaintenance" "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarysnapshot" postgresstore "knowledge-governance/internal/platform/postgres" ) // 目标内容在窗口内的几种"已知状态",决定目标能否被重新开放: // - contentUntouched:这次窗口还没写过目标(可以安全开放) // - contentSucceeded / contentRolledBack:内容已验证,或已回滚并验证(可以开放) // - contentHalfWritten / contentRollbackFails:内容可能半写(必须保持维护) const ( contentUntouched = "" contentSucceeded = "succeeded" contentRolledBack = "rolled_back" contentHalfWritten = "half_written" contentRollbackFails = "rollback_failed" ) // contentTrackedTarget 只做一件事:在第一次写目标时把"内容有风险"标出来。 // 这样"授权过期/进程被杀/校验失败"这些早退路径不会把半写目标当成没动过。 type contentTrackedTarget struct { inner libraryrelease.Target onApply func() } func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) { return t.inner.Current(ctx) } func (t *contentTrackedTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error { t.onApply() return t.inner.Apply(ctx, stage, snapshotID) } func (t *contentTrackedTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error { return t.inner.Verify(ctx, expect) } func (t *contentTrackedTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error { return t.inner.Restore(ctx, backup) } // libraryReleasePayload 是同步与恢复共用的队列载荷形状。 type libraryReleasePayload struct { LibraryJobID string `json:"library_job_id"` Scope string `json:"scope"` TargetSnapshotID string `json:"target_snapshot_id"` // FinalizeOnly:只做收尾(校验 + 登记生效版本),不重写目标。 // 用于"写入已完成但版本登记失败"的续作,避免把已生效的生产再写一遍。 FinalizeOnly bool `json:"finalize_only,omitempty"` } // openLibraryMaintenance 按部署注入的配置装配实例级维护目标。 // 未接线时返回明确原因:只锁治理库 scope 不算维护,宁可拒绝写也不冒充。 func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) { database, err := librarymaintenance.DatabaseIdentity(config.LibraryProductionDatabaseURL) if err != nil { return nil, err } return librarymaintenance.New(librarymaintenance.Config{ Instance: config.LibraryTargetInstance, Project: config.LibraryTargetProject, Writers: config.LibraryTargetWriters, NonWriters: config.LibraryTargetNonWriters, DockerSocket: config.LibraryTargetDockerSocket, IngressURL: config.LibraryTargetIngressURL, IngressService: config.LibraryTargetIngressService, IngressDrainFile: config.LibraryTargetIngressDrainFile, Database: database, Index: config.LibraryProductionIndex, Bucket: config.LibraryProductionS3Bucket, // 后台静默检查的运行时依赖:HTTP 排空不等于后台任务排空。 DatabaseURL: config.LibraryProductionDatabaseURL, RedisAddr: config.LibraryTargetRedisAddr, RedisPassword: config.LibraryTargetRedisPassword, RedisDB: config.LibraryTargetRedisDB, }) } // runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 → // 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。 // 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。 func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job, kind governance.LibraryJobKind) error { var payload libraryReleasePayload if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" { return fmt.Errorf("invalid library %s payload", kind) } now := time.Now().UTC() _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now) recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope} // 四类进度的真实总量来自快照清单(manifest):读不到就保持 NULL,不写 0。 if archive, openErr := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024); openErr == nil { if manifest, loadErr := archive.Load(payload.TargetSnapshotID); loadErr == nil { recorder.totals = stageSizesFrom(manifest) } archive.Close() } // 崩溃接管:作业里已经持久化过预备份快照,就直接复用它继续, // 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。 previous, err := store.LibraryJob(ctx, payload.LibraryJobID) resumed := err == nil && previous.SnapshotID != "" if resumed { recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID) } gateReasons := make([]string, 0, 3) if !config.ProductionReleaseRequestsEnabled { gateReasons = append(gateReasons, "生产写入闸口关闭") } if !config.ProductionBackupRestoreVerified { gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据") } if len(gateReasons) > 0 { message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";") recorder.Stage(ctx, "prebackup", "failed", message) // 接管场景下生产可能已经是半写状态:这时"无需恢复"是错的, // 必须把既有预备份作为恢复基线留住,并保持目标被占用直到处理完。 recovery := "未对生产执行任何写入,无需恢复" if resumed { recovery = "生产可能处于半写状态,必须先按预备份 " + previous.SnapshotID + " 恢复后再重试" } _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message, recovery, previous.SnapshotID, "", time.Now().UTC()) return errors.New(message) } digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID}) if err != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "无法计算请求摘要", "prebackup") } grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now) if err != nil || !found { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "没有与目标快照匹配的有效发布授权", "prebackup") } // 生产索引映射基线:优先当前生效版本,其次任何一次成功预备份(它也记录了生产实际映射)。 // 只认"生效版本"会在恢复把版本 supersede 掉之后误判"索引被改动"。 knownMapping := "" if versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50); versionErr == nil { for _, item := range versions { if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil { if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" { knownMapping = raw break } } } if knownMapping == "" { for _, item := range versions { if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" { knownMapping = raw break } } } } target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping) if wireErr == nil { if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { recorder.mappingDigest = mapping } } if wireErr != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup") } defer target.Close() // 维护目标必须在真正动目标之前确定:缺配置就拒绝写,不用"库级锁"顶替。 manager, managerErr := openLibraryMaintenance(config) if managerErr != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup") } defer manager.Close() // 拓扑必须先过关:写入者发布了端口 = 存在绕闸直连,窗口关的只是半扇门。 if topoErr := manager.VerifyTopology(ctx); topoErr != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup") } targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope, redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket) originalState, stateErr := manager.OriginalState(ctx) if stateErr != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup") } identity := manager.Target() // 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。 instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{ Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope, TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance, Services: identity.Writers, IngressState: "unknown", State: "acquired", OriginalServiceState: originalState, Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放", }, now) if instanceErr != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "prebackup") } _ = instanceLock maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{ Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity + " | " + manager.Description(), Level: librarymaintenance.LevelInstance, TargetFingerprint: identity.Fingerprint, Services: identity.Writers, IngressState: "unknown", State: "acquired", OriginalServiceState: originalState, Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放", }, now) if acquireErr != nil { _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", "scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now) return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup") } _ = maintenance coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot) if openErr != nil { // 协调器都打不开:这一次没动过目标。接管场景除外——上一次尝试可能已经把 // 写入者停住并写了一半,那种情况下保持占用是唯一安全的选择。 if resumed { _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", "维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC()) _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", "维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC()) } else { _ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, "未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC()) } return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "维护协调器不可用:"+openErr.Error(), "prebackup") } defer control.Close() defer coordinator.Close() // 接管场景下目标可能已经被上一次尝试写过:在证明"已回滚或已验证"之前, // 一律按半写对待,任何早退都不许重新开放目标。 contentState := contentUntouched if resumed { contentState = contentHalfWritten } tracked := &contentTrackedTarget{inner: target, onApply: func() { contentState = contentHalfWritten }} allowRestore := func(error) bool { return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack } execute := func(ctx context.Context) error { if payload.FinalizeOnly { // 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。 recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致") if verifyErr := tracked.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil { contentState = contentHalfWritten message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error() recorder.Stage(ctx, "finalize", "failed", message) _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) return errors.New(message) } contentState = contentSucceeded mappingDigest, _ := target.Search().MappingSHA256(ctx) if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker", Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only", "index_mapping_sha256": mappingDigest}, CreatedAt: time.Now().UTC(), }); err != nil { message := "收尾续作仍无法登记生效版本:" + err.Error() recorder.Stage(ctx, "finalize", "failed", message) _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) return errors.New(message) } recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)") _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) return nil } var backupSource libraryrelease.BackupStore = backups if resumed { backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID, SnapshotID: previous.SnapshotID}} } request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now, Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID, ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}} _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now) outcome := libraryrelease.Run(ctx, request, tracked, backupSource, recorder) finishedAt := time.Now().UTC() switch outcome.State { case "succeeded": contentState = contentSucceeded mappingDigest := "" if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { mappingDigest = mapping } // 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。 // 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。 recorder.Stage(ctx, "finalize", "running", "登记生产生效版本") versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy, Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID, "backup_bytes": outcome.Backup.Bytes, "action": string(kind), "index_mapping_sha256": mappingDigest}, CreatedAt: finishedAt, }) if versionErr != nil { message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error() recorder.Stage(ctx, "finalize", "failed", message) // 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。 if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记", outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr) } return errors.New(message) } recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本") if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { // 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。 return fmt.Errorf("作业状态写入失败: %w", advanceErr) } log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind, payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID) return nil case "rolled_back": contentState = contentRolledBack _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error, outcome.Rollback, "", "", finishedAt) return errors.New(outcome.Error) case "rollback_failed": contentState = contentRollbackFails _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error, outcome.Rollback, "", "", finishedAt) return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback) default: recovery := outcome.Recovery if recovery == "" { recovery = "未对生产执行任何写入,无需恢复" } _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt) return errors.New(outcome.Error) } } prepare := func(ctx context.Context) error { // 停下写入者之前必须确认目标现在是健康的:拿一个已经不健康的目标开窗口, // 只会把"本来坏的"记成"维护造成的"。 return manager.VerifyRunning(ctx) } windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) { // 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘:没有预备份身份的 // 窗口之后既不能自动恢复,也不许被任何收尾路径开放。 recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error { if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil { return err } return w.MarkContentRisk(backupCtx) } if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed", "目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil { // 记录失败不影响隔离本身,但必须留痕,不能假装记上了。 log.Printf("library %s maintenance ingress record failed: %v", payload.LibraryJobID, ingressErr) } return "", execute(ctx) } window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore) // 收尾:只有"内容已验证/已回滚"且服务与入口都恢复,才释放维护占用。 now2 := time.Now().UTC() switch { case window.State == "held": note := "目标保持维护:内容状态未验证(半写或回滚失败),或服务/入口未能恢复" if windowErr != nil { note += ":" + windowErr.Error() } _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2) _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2) if windowErr == nil { return errors.New(note) } return windowErr case contentState == contentSucceeded: if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, "写入与校验完成,目标已按原运行状态恢复", now2); err != nil { return fmt.Errorf("目标已恢复但维护占用释放失败: %w", err) } return windowErr case contentState == contentRolledBack: if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, "本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2); err != nil { return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err) } return windowErr default: // 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。 if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, "未写目标,维护窗口未开启", now2); err != nil { return fmt.Errorf("维护占用释放失败: %w", err) } return windowErr } } // releaseWindowLocks 是唯一释放路径:先记录入口已恢复,再放 scope 行与实例锁。 // 顺序反过来会出现"实例锁放了、scope 还挂着"的中间态,页面就会说正在维护。 func releaseWindowLocks(ctx context.Context, store *postgresstore.Store, instance, scope, jobID, note string, now time.Time) error { _ = store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now) _ = store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now) _ = store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now) _ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now) if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); err != nil { return err } return store.ReleaseLibraryMaintenance(ctx, scope, jobID, now) } // resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业: // 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用), // 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。 func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Store, config appconfig.Config) (int, int, error) { interrupted, err := store.InterruptedLibraryJobs(ctx) if err != nil { return 0, 0, err } resumed, failed := 0, 0 for _, job := range interrupted { now := time.Now().UTC() if job.SnapshotID == "" { message := "worker 中断时还没有完成预备份,不能盲写目标" _ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message, "未对生产执行写入", "", "", now) failed++ continue } // 原子抢占:多个 worker 同时接管也只有一个能把 running 改成 pending。 claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now) if claimErr != nil { log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr) continue } if !claimed { continue } queueType := librarySyncJobType if job.Kind == governance.LibraryJobRestore { queueType = libraryRestoreJobType } payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope, "target_snapshot_id": job.TargetSnapshotID}) // 队列作业 ID 与幂等键都按作业固定:重复接管/重复点击只会得到同一个队列作业。 queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{ ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload, AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + job.ID, }, now) if err != nil || queueJob == nil { // 队列作业 ID 是确定性的:并发接管时另一个 worker 已经建好了, // 这里按"已存在"处理并继续接管,不去覆盖别人的队列行。 if err != nil && strings.Contains(err.Error(), "duplicate key") { queueID := governance.QueueJobIDFor(job.ID) if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil { log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr) continue } if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil { log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr) } else { resumed++ } continue } log.Printf("library job %s resume enqueue failed: %v", job.ID, err) continue } if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil { log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr) continue } if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil { log.Printf("library job %s resume attach failed: %v", job.ID, attachErr) continue } resumed++ log.Printf("library job %s resumed after worker restart (backup %s)", job.ID, job.SnapshotID) } return resumed, failed, nil } // fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。 type fixedBackupStore struct{ ref libraryrelease.BackupRef } func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) { return s.ref, nil } // BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误: // 上游据此在写目标前中止,保证"没有可恢复基线就不写"。 func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error { now := time.Now().UTC() if r.onBackup != nil { if err := r.onBackup(ctx, backup); err != nil { return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err) } } if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "", backup.SnapshotID, "", now); err != nil { return fmt.Errorf("持久化预备份引用失败: %w", err) } if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID), Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID, SourceJobID: r.jobID, CreatedBy: "worker", Evidence: map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes, "job": r.jobID, "index_mapping_sha256": r.mappingDigest}, CreatedAt: now, }); err != nil { return fmt.Errorf("登记预备份版本失败: %w", err) } return nil } func failLibraryJob(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder, jobID, message, stage string) error { recorder.Stage(ctx, stage, "failed", message) _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message, "未对生产执行任何写入,无需恢复", "", "", time.Now().UTC()) return errors.New(message) } func kindLabel(kind governance.LibraryJobKind) string { if kind == governance.LibraryJobRestore { return "恢复" } return "同步" } func digestKey(kind governance.LibraryJobKind) string { if kind == governance.LibraryJobRestore { return "restore_snapshot_id" } return "target_snapshot_id" }