package main import ( "context" "encoding/json" "errors" "fmt" "log" "strings" "time" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryrelease" postgresstore "knowledge-governance/internal/platform/postgres" ) // libraryReleasePayload 是同步与恢复共用的队列载荷形状。 type libraryReleasePayload struct { LibraryJobID string `json:"library_job_id"` Scope string `json:"scope"` TargetSnapshotID string `json:"target_snapshot_id"` // FinalizeOnly:只做收尾(校验 + 登记生效版本),不重写目标。 // 用于"写入已完成但版本登记失败"的续作,避免把已生效的生产再写一遍。 FinalizeOnly bool `json:"finalize_only,omitempty"` } // runLibraryRelease 执行"闸口 → 授权 → 预备份 → 四类写入 → 校验"的完整链路。 // 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。 func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job, kind governance.LibraryJobKind) error { var payload libraryReleasePayload if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" { return fmt.Errorf("invalid library %s payload", kind) } now := time.Now().UTC() _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now) recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope} // 崩溃接管:作业里已经持久化过预备份快照,就直接复用它继续, // 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。 previous, err := store.LibraryJob(ctx, payload.LibraryJobID) resumed := err == nil && previous.SnapshotID != "" if resumed { recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID) } gateReasons := make([]string, 0, 3) if !config.ProductionReleaseRequestsEnabled { gateReasons = append(gateReasons, "生产写入闸口关闭") } if !config.ProductionBackupRestoreVerified { gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据") } if len(gateReasons) > 0 { message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";") recorder.Stage(ctx, "prebackup", "failed", message) // 接管场景下生产可能已经是半写状态:这时"无需恢复"是错的, // 必须把既有预备份作为恢复基线留住,并保持目标被占用直到处理完。 recovery := "未对生产执行任何写入,无需恢复" if resumed { recovery = "生产可能处于半写状态,必须先按预备份 " + previous.SnapshotID + " 恢复后再重试" } _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message, recovery, previous.SnapshotID, "", time.Now().UTC()) return errors.New(message) } digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID}) if err != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "无法计算请求摘要", "prebackup") } grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now) if err != nil || !found { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "没有与目标快照匹配的有效发布授权", "prebackup") } // 生产索引映射基线:优先当前生效版本,其次任何一次成功预备份(它也记录了生产实际映射)。 // 只认"生效版本"会在恢复把版本 supersede 掉之后误判"索引被改动"。 knownMapping := "" if versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50); versionErr == nil { for _, item := range versions { if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil { if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" { knownMapping = raw break } } } if knownMapping == "" { for _, item := range versions { if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" { knownMapping = raw break } } } } target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping) if wireErr == nil { if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { recorder.mappingDigest = digest } } if wireErr != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup") } defer target.Close() if payload.FinalizeOnly { // 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。 recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致") if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil { message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error() recorder.Stage(ctx, "finalize", "failed", message) _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) return errors.New(message) } mappingDigest, _ := target.Search().MappingSHA256(ctx) if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker", Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only", "index_mapping_sha256": mappingDigest}, CreatedAt: time.Now().UTC(), }); err != nil { message := "收尾续作仍无法登记生效版本:" + err.Error() recorder.Stage(ctx, "finalize", "failed", message) _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) return errors.New(message) } recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)") _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) return nil } var backupSource libraryrelease.BackupStore = backups if resumed { backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID, SnapshotID: previous.SnapshotID}} } request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now, Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID, ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}} outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder) finishedAt := time.Now().UTC() switch outcome.State { case "succeeded": mappingDigest := "" if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { mappingDigest = digest } // 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。 // 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。 recorder.Stage(ctx, "finalize", "running", "登记生产生效版本") versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy, Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID, "backup_bytes": outcome.Backup.Bytes, "action": string(kind), "index_mapping_sha256": mappingDigest}, CreatedAt: finishedAt, }) if versionErr != nil { message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error() recorder.Stage(ctx, "finalize", "failed", message) // 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。 if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记", outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr) } return errors.New(message) } recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本") if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { // 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。 return fmt.Errorf("作业状态写入失败: %w", advanceErr) } log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind, payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID) return nil case "rolled_back": _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error, outcome.Rollback, "", "", finishedAt) return errors.New(outcome.Error) case "rollback_failed": _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error, outcome.Rollback, "", "", finishedAt) return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback) default: recovery := outcome.Recovery if recovery == "" { recovery = "未对生产执行任何写入,无需恢复" } _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt) return errors.New(outcome.Error) } } // resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业: // 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用), // 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。 func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Store, config appconfig.Config) (int, int, error) { interrupted, err := store.InterruptedLibraryJobs(ctx) if err != nil { return 0, 0, err } resumed, failed := 0, 0 for _, job := range interrupted { now := time.Now().UTC() if job.SnapshotID == "" { message := "worker 中断时还没有完成预备份,不能盲写目标" _ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message, "未对生产执行写入", "", "", now) failed++ continue } // 原子抢占:多个 worker 同时接管也只有一个能把 running 改成 pending。 claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now) if claimErr != nil { log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr) continue } if !claimed { continue } queueType := librarySyncJobType if job.Kind == governance.LibraryJobRestore { queueType = libraryRestoreJobType } payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope, "target_snapshot_id": job.TargetSnapshotID}) // 队列作业 ID 与幂等键都按作业固定:重复接管/重复点击只会得到同一个队列作业。 queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{ ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload, AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + job.ID, }, now) if err != nil || queueJob == nil { // 队列作业 ID 是确定性的:并发接管时另一个 worker 已经建好了, // 这里按"已存在"处理并继续接管,不去覆盖别人的队列行。 if err != nil && strings.Contains(err.Error(), "duplicate key") { queueID := governance.QueueJobIDFor(job.ID) if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil { log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr) continue } if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil { log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr) } else { resumed++ } continue } log.Printf("library job %s resume enqueue failed: %v", job.ID, err) continue } if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil { log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr) continue } if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil { log.Printf("library job %s resume attach failed: %v", job.ID, attachErr) continue } resumed++ log.Printf("library job %s resumed after worker restart (backup %s)", job.ID, job.SnapshotID) } return resumed, failed, nil } // fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。 type fixedBackupStore struct{ ref libraryrelease.BackupRef } func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) { return s.ref, nil } // BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误: // 上游据此在写目标前中止,保证"没有可恢复基线就不写"。 func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error { now := time.Now().UTC() if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "", backup.SnapshotID, "", now); err != nil { return fmt.Errorf("持久化预备份引用失败: %w", err) } if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID), Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID, SourceJobID: r.jobID, CreatedBy: "worker", Evidence: map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes, "job": r.jobID, "index_mapping_sha256": r.mappingDigest}, CreatedAt: now, }); err != nil { return fmt.Errorf("登记预备份版本失败: %w", err) } return nil } func failLibraryJob(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder, jobID, message, stage string) error { recorder.Stage(ctx, stage, "failed", message) _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message, "未对生产执行任何写入,无需恢复", "", "", time.Now().UTC()) return errors.New(message) } func kindLabel(kind governance.LibraryJobKind) string { if kind == governance.LibraryJobRestore { return "恢复" } return "同步" } func digestKey(kind governance.LibraryJobKind) string { if kind == governance.LibraryJobRestore { return "restore_snapshot_id" } return "target_snapshot_id" }