Files
knowledge-governance/cmd/worker/library_handlers.go
T
姚章浩 5afe151c7d
build / build (push) Successful in 1m14s
fix(library): 维护目标把后台静默检查的依赖接上
openLibraryMaintenance 漏传了目标库/队列,导致实例级窗口永远报
"缺少后台静默检查"——接线漏了,防线却看起来像在工作。
2026-10-29 01:01:52 +08:00

569 lines
28 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"strings"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/librarymaintenance"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// 目标内容在窗口内的几种"已知状态",决定目标能否被重新开放:
// - contentUntouched:这次窗口还没写过目标(可以安全开放)
// - contentSucceeded / contentRolledBack:内容已验证,或已回滚并验证(可以开放)
// - contentHalfWritten / contentRollbackFails:内容可能半写(必须保持维护)
const (
contentUntouched = ""
contentSucceeded = "succeeded"
contentRolledBack = "rolled_back"
contentHalfWritten = "half_written"
contentRollbackFails = "rollback_failed"
)
// contentTrackedTarget 只做一件事:在第一次写目标时把"内容有风险"标出来。
// 这样"授权过期/进程被杀/校验失败"这些早退路径不会把半写目标当成没动过。
type contentTrackedTarget struct {
inner libraryrelease.Target
onApply func()
}
func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) {
return t.inner.Current(ctx)
}
func (t *contentTrackedTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error {
t.onApply()
return t.inner.Apply(ctx, stage, snapshotID)
}
func (t *contentTrackedTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error {
return t.inner.Verify(ctx, expect)
}
func (t *contentTrackedTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error {
return t.inner.Restore(ctx, backup)
}
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
type libraryReleasePayload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
TargetSnapshotID string `json:"target_snapshot_id"`
// FinalizeOnly:只做收尾(校验 + 登记生效版本),不重写目标。
// 用于"写入已完成但版本登记失败"的续作,避免把已生效的生产再写一遍。
FinalizeOnly bool `json:"finalize_only,omitempty"`
}
// openLibraryMaintenance 按部署注入的配置装配实例级维护目标。
// 未接线时返回明确原因:只锁治理库 scope 不算维护,宁可拒绝写也不冒充。
func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) {
database, err := librarymaintenance.DatabaseIdentity(config.LibraryProductionDatabaseURL)
if err != nil {
return nil, err
}
return librarymaintenance.New(librarymaintenance.Config{
Instance: config.LibraryTargetInstance,
Project: config.LibraryTargetProject,
Writers: config.LibraryTargetWriters,
NonWriters: config.LibraryTargetNonWriters,
DockerSocket: config.LibraryTargetDockerSocket,
IngressURL: config.LibraryTargetIngressURL,
IngressService: config.LibraryTargetIngressService,
IngressDrainFile: config.LibraryTargetIngressDrainFile,
Database: database,
Index: config.LibraryProductionIndex,
Bucket: config.LibraryProductionS3Bucket,
// 后台静默检查的运行时依赖:HTTP 排空不等于后台任务排空。
DatabaseURL: config.LibraryProductionDatabaseURL,
RedisAddr: config.LibraryTargetRedisAddr,
RedisPassword: config.LibraryTargetRedisPassword,
RedisDB: config.LibraryTargetRedisDB,
})
}
// runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 →
// 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。
// 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。
func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config appconfig.Config,
job jobs.Job, kind governance.LibraryJobKind) error {
var payload libraryReleasePayload
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" {
return fmt.Errorf("invalid library %s payload", kind)
}
now := time.Now().UTC()
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope}
// 四类进度的真实总量来自快照清单(manifest):读不到就保持 NULL,不写 0。
if archive, openErr := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024); openErr == nil {
if manifest, loadErr := archive.Load(payload.TargetSnapshotID); loadErr == nil {
recorder.totals = stageSizesFrom(manifest)
}
archive.Close()
}
// 崩溃接管:作业里已经持久化过预备份快照,就直接复用它继续,
// 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。
previous, err := store.LibraryJob(ctx, payload.LibraryJobID)
resumed := err == nil && previous.SnapshotID != ""
if resumed {
recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID)
}
gateReasons := make([]string, 0, 3)
if !config.ProductionReleaseRequestsEnabled {
gateReasons = append(gateReasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
}
if len(gateReasons) > 0 {
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
recorder.Stage(ctx, "prebackup", "failed", message)
// 接管场景下生产可能已经是半写状态:这时"无需恢复"是错的,
// 必须把既有预备份作为恢复基线留住,并保持目标被占用直到处理完。
recovery := "未对生产执行任何写入,无需恢复"
if resumed {
recovery = "生产可能处于半写状态,必须先按预备份 " + previous.SnapshotID + " 恢复后再重试"
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
recovery, previous.SnapshotID, "", time.Now().UTC())
return errors.New(message)
}
digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID})
if err != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "无法计算请求摘要", "prebackup")
}
grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now)
if err != nil || !found {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "没有与目标快照匹配的有效发布授权", "prebackup")
}
// 生产索引映射基线:优先当前生效版本,其次任何一次成功预备份(它也记录了生产实际映射)。
// 只认"生效版本"会在恢复把版本 supersede 掉之后误判"索引被改动"。
knownMapping := ""
if versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50); versionErr == nil {
for _, item := range versions {
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
knownMapping = raw
break
}
}
}
if knownMapping == "" {
for _, item := range versions {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
knownMapping = raw
break
}
}
}
}
target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping)
if wireErr == nil {
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
recorder.mappingDigest = mapping
}
}
if wireErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup")
}
defer target.Close()
// 维护目标必须在真正动目标之前确定:缺配置就拒绝写,不用"库级锁"顶替。
manager, managerErr := openLibraryMaintenance(config)
if managerErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup")
}
defer manager.Close()
// 拓扑必须先过关:写入者发布了端口 = 存在绕闸直连,窗口关的只是半扇门。
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup")
}
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
originalState, stateErr := manager.OriginalState(ctx)
if stateErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup")
}
identity := manager.Target()
// 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。
instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope,
TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance,
Services: identity.Writers, IngressState: "unknown", State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
}, now)
if instanceErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "prebackup")
}
_ = instanceLock
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID,
TargetIdentity: targetIdentity + " | " + manager.Description(),
Level: librarymaintenance.LevelInstance,
TargetFingerprint: identity.Fingerprint,
Services: identity.Writers,
IngressState: "unknown",
State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
}, now)
if acquireErr != nil {
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
"scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now)
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup")
}
_ = maintenance
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
// 协调器都打不开:这一次没动过目标。接管场景除外——上一次尝试可能已经把
// 写入者停住并写了一半,那种情况下保持占用是唯一安全的选择。
if resumed {
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
} else {
_ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC())
}
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "维护协调器不可用:"+openErr.Error(), "prebackup")
}
defer control.Close()
defer coordinator.Close()
// 接管场景下目标可能已经被上一次尝试写过:在证明"已回滚或已验证"之前,
// 一律按半写对待,任何早退都不许重新开放目标。
contentState := contentUntouched
if resumed {
contentState = contentHalfWritten
}
tracked := &contentTrackedTarget{inner: target, onApply: func() { contentState = contentHalfWritten }}
allowRestore := func(error) bool {
return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack
}
execute := func(ctx context.Context) error {
if payload.FinalizeOnly {
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
if verifyErr := tracked.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil {
contentState = contentHalfWritten
message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
contentState = contentSucceeded
mappingDigest, _ := target.Search().MappingSHA256(ctx)
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker",
Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only",
"index_mapping_sha256": mappingDigest},
CreatedAt: time.Now().UTC(),
}); err != nil {
message := "收尾续作仍无法登记生效版本:" + err.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return nil
}
var backupSource libraryrelease.BackupStore = backups
if resumed {
backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
SnapshotID: previous.SnapshotID}}
}
request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID,
ParamsDigest: digest, Now: now,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
outcome := libraryrelease.Run(ctx, request, tracked, backupSource, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
case "succeeded":
contentState = contentSucceeded
mappingDigest := ""
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
mappingDigest = mapping
}
// 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。
// 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。
recorder.Stage(ctx, "finalize", "running", "登记生产生效版本")
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
Scope: payload.Scope,
Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID,
SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy,
Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID,
"backup_bytes": outcome.Backup.Bytes, "action": string(kind),
"index_mapping_sha256": mappingDigest},
CreatedAt: finishedAt,
})
if versionErr != nil {
message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
// 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr)
}
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本")
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
// 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。
return fmt.Errorf("作业状态写入失败: %w", advanceErr)
}
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID)
return nil
case "rolled_back":
contentState = contentRolledBack
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
contentState = contentRollbackFails
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
default:
recovery := outcome.Recovery
if recovery == "" {
recovery = "未对生产执行任何写入,无需恢复"
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup",
outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt)
return errors.New(outcome.Error)
}
}
prepare := func(ctx context.Context) error {
// 停下写入者之前必须确认目标现在是健康的:拿一个已经不健康的目标开窗口,
// 只会把"本来坏的"记成"维护造成的"。
return manager.VerifyRunning(ctx)
}
windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) {
// 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘:没有预备份身份的
// 窗口之后既不能自动恢复,也不许被任何收尾路径开放。
recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error {
if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil {
return err
}
return w.MarkContentRisk(backupCtx)
}
if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed",
"目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil {
// 记录失败不影响隔离本身,但必须留痕,不能假装记上了。
log.Printf("library %s maintenance ingress record failed: %v", payload.LibraryJobID, ingressErr)
}
return "", execute(ctx)
}
window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore)
// 收尾:只有"内容已验证/已回滚"且服务与入口都恢复,才释放维护占用。
now2 := time.Now().UTC()
switch {
case window.State == "held":
note := "目标保持维护:内容状态未验证(半写或回滚失败),或服务/入口未能恢复"
if windowErr != nil {
note += ":" + windowErr.Error()
}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
if windowErr == nil {
return errors.New(note)
}
return windowErr
case contentState == contentSucceeded:
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"写入与校验完成,目标已按原运行状态恢复", now2); err != nil {
return fmt.Errorf("目标已恢复但维护占用释放失败: %w", err)
}
return windowErr
case contentState == contentRolledBack:
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2); err != nil {
return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err)
}
return windowErr
default:
// 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"未写目标,维护窗口未开启", now2); err != nil {
return fmt.Errorf("维护占用释放失败: %w", err)
}
return windowErr
}
}
// releaseWindowLocks 是唯一释放路径:先记录入口已恢复,再放 scope 行与实例锁。
// 顺序反过来会出现"实例锁放了、scope 还挂着"的中间态,页面就会说正在维护。
func releaseWindowLocks(ctx context.Context, store *postgresstore.Store, instance, scope, jobID, note string, now time.Time) error {
_ = store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now)
_ = store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now)
_ = store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now)
_ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now)
if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); err != nil {
return err
}
return store.ReleaseLibraryMaintenance(ctx, scope, jobID, now)
}
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业:
// 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用),
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。
func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Store, config appconfig.Config) (int, int, error) {
interrupted, err := store.InterruptedLibraryJobs(ctx)
if err != nil {
return 0, 0, err
}
resumed, failed := 0, 0
for _, job := range interrupted {
now := time.Now().UTC()
if job.SnapshotID == "" {
message := "worker 中断时还没有完成预备份,不能盲写目标"
_ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行写入", "", "", now)
failed++
continue
}
// 原子抢占:多个 worker 同时接管也只有一个能把 running 改成 pending。
claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now)
if claimErr != nil {
log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr)
continue
}
if !claimed {
continue
}
queueType := librarySyncJobType
if job.Kind == governance.LibraryJobRestore {
queueType = libraryRestoreJobType
}
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
"target_snapshot_id": job.TargetSnapshotID})
// 队列作业 ID 与幂等键都按作业固定:重复接管/重复点击只会得到同一个队列作业。
queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload,
AvailableAt: now, MaxAttempts: 1,
IdempotencyKey: "library.resume.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
// 队列作业 ID 是确定性的:并发接管时另一个 worker 已经建好了,
// 这里按"已存在"处理并继续接管,不去覆盖别人的队列行。
if err != nil && strings.Contains(err.Error(), "duplicate key") {
queueID := governance.QueueJobIDFor(job.ID)
if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil {
log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr)
continue
}
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil {
log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr)
} else {
resumed++
}
continue
}
log.Printf("library job %s resume enqueue failed: %v", job.ID, err)
continue
}
if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil {
log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr)
continue
}
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil {
log.Printf("library job %s resume attach failed: %v", job.ID, attachErr)
continue
}
resumed++
log.Printf("library job %s resumed after worker restart (backup %s)", job.ID, job.SnapshotID)
}
return resumed, failed, nil
}
// fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。
type fixedBackupStore struct{ ref libraryrelease.BackupRef }
func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
return s.ref, nil
}
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误:
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
now := time.Now().UTC()
if r.onBackup != nil {
if err := r.onBackup(ctx, backup); err != nil {
return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err)
}
}
if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "",
backup.SnapshotID, "", now); err != nil {
return fmt.Errorf("持久化预备份引用失败: %w", err)
}
if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID),
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID,
SourceJobID: r.jobID, CreatedBy: "worker",
Evidence: map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes,
"job": r.jobID, "index_mapping_sha256": r.mappingDigest},
CreatedAt: now,
}); err != nil {
return fmt.Errorf("登记预备份版本失败: %w", err)
}
return nil
}
func failLibraryJob(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder, jobID, message, stage string) error {
recorder.Stage(ctx, stage, "failed", message)
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message,
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
return errors.New(message)
}
func kindLabel(kind governance.LibraryJobKind) string {
if kind == governance.LibraryJobRestore {
return "恢复"
}
return "同步"
}
func digestKey(kind governance.LibraryJobKind) string {
if kind == governance.LibraryJobRestore {
return "restore_snapshot_id"
}
return "target_snapshot_id"
}