c72b71d78a
build / build (push) Failing after 17s
为什么:父审指出预备份引用持久化失败仍继续写目标、接管依赖启动扫描且漏掉租约未过期、 恢复失败被标成"无需恢复"、版本登记与作业状态分叉。 - BackupPrepared 返回 error,持久化失败即零写入;预备份同时登记 production_prebackup 版本。 - 接管改为按队列租约判定 + 运行期 30s 扫描 + 原子抢占 + 固定幂等键 + attempt 归零。 - 接管场景保留预备份、不再宣称"未写无需恢复"。 - 恢复/重放同一快照重新激活 production_effective(清 superseded_at)。 - finalize 续作只做校验+登记(finalize_only),不重写生产。 - 索引映射基线缺失时采用首次观察值并登记;索引校验加孤儿文档检查(按 knowledge_id)。 真实验收(候选 API :18089 + 真实 PG/ES/MinIO): A0 首次发布、A1 撤销、B 按版本恢复、C 写阶段 SIGKILL 接管、 D 跨 kind 互斥与多 worker 单次执行、E1 登记失败、E2 finalize 续作、E3 预备份失败零写入。
330 lines
16 KiB
Go
330 lines
16 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
)
|
||
|
||
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
|
||
type libraryReleasePayload struct {
|
||
LibraryJobID string `json:"library_job_id"`
|
||
Scope string `json:"scope"`
|
||
TargetSnapshotID string `json:"target_snapshot_id"`
|
||
// FinalizeOnly:只做收尾(校验 + 登记生效版本),不重写目标。
|
||
// 用于"写入已完成但版本登记失败"的续作,避免把已生效的生产再写一遍。
|
||
FinalizeOnly bool `json:"finalize_only,omitempty"`
|
||
}
|
||
|
||
// runLibraryRelease 执行"闸口 → 授权 → 预备份 → 四类写入 → 校验"的完整链路。
|
||
// 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。
|
||
func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config appconfig.Config,
|
||
job jobs.Job, kind governance.LibraryJobKind) error {
|
||
var payload libraryReleasePayload
|
||
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" {
|
||
return fmt.Errorf("invalid library %s payload", kind)
|
||
}
|
||
now := time.Now().UTC()
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
|
||
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope}
|
||
|
||
// 崩溃接管:作业里已经持久化过预备份快照,就直接复用它继续,
|
||
// 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。
|
||
previous, err := store.LibraryJob(ctx, payload.LibraryJobID)
|
||
resumed := err == nil && previous.SnapshotID != ""
|
||
if resumed {
|
||
recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID)
|
||
}
|
||
|
||
gateReasons := make([]string, 0, 3)
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
gateReasons = append(gateReasons, "生产写入闸口关闭")
|
||
}
|
||
if !config.ProductionBackupRestoreVerified {
|
||
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
|
||
}
|
||
if len(gateReasons) > 0 {
|
||
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
|
||
recorder.Stage(ctx, "prebackup", "failed", message)
|
||
// 接管场景下生产可能已经是半写状态:这时"无需恢复"是错的,
|
||
// 必须把既有预备份作为恢复基线留住,并保持目标被占用直到处理完。
|
||
recovery := "未对生产执行任何写入,无需恢复"
|
||
if resumed {
|
||
recovery = "生产可能处于半写状态,必须先按预备份 " + previous.SnapshotID + " 恢复后再重试"
|
||
}
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
|
||
recovery, previous.SnapshotID, "", time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
|
||
digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID})
|
||
if err != nil {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "无法计算请求摘要", "prebackup")
|
||
}
|
||
grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now)
|
||
if err != nil || !found {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "没有与目标快照匹配的有效发布授权", "prebackup")
|
||
}
|
||
// 生产索引映射基线:优先当前生效版本,其次任何一次成功预备份(它也记录了生产实际映射)。
|
||
// 只认"生效版本"会在恢复把版本 supersede 掉之后误判"索引被改动"。
|
||
knownMapping := ""
|
||
if versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50); versionErr == nil {
|
||
for _, item := range versions {
|
||
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
|
||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||
knownMapping = raw
|
||
break
|
||
}
|
||
}
|
||
}
|
||
if knownMapping == "" {
|
||
for _, item := range versions {
|
||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||
knownMapping = raw
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping)
|
||
if wireErr == nil {
|
||
if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
|
||
recorder.mappingDigest = digest
|
||
}
|
||
}
|
||
if wireErr != nil {
|
||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup")
|
||
}
|
||
defer target.Close()
|
||
if payload.FinalizeOnly {
|
||
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
|
||
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
|
||
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil {
|
||
message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
|
||
"生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
mappingDigest, _ := target.Search().MappingSHA256(ctx)
|
||
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
|
||
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
|
||
SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker",
|
||
Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only",
|
||
"index_mapping_sha256": mappingDigest},
|
||
CreatedAt: time.Now().UTC(),
|
||
}); err != nil {
|
||
message := "收尾续作仍无法登记生效版本:" + err.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
|
||
"预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
|
||
previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||
return nil
|
||
}
|
||
|
||
var backupSource libraryrelease.BackupStore = backups
|
||
if resumed {
|
||
backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
|
||
SnapshotID: previous.SnapshotID}}
|
||
}
|
||
|
||
request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID,
|
||
ParamsDigest: digest, Now: now,
|
||
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
|
||
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
|
||
outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder)
|
||
finishedAt := time.Now().UTC()
|
||
switch outcome.State {
|
||
case "succeeded":
|
||
mappingDigest := ""
|
||
if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
|
||
mappingDigest = digest
|
||
}
|
||
// 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。
|
||
// 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。
|
||
recorder.Stage(ctx, "finalize", "running", "登记生产生效版本")
|
||
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
|
||
Scope: payload.Scope,
|
||
Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID,
|
||
SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy,
|
||
Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID,
|
||
"backup_bytes": outcome.Backup.Bytes, "action": string(kind),
|
||
"index_mapping_sha256": mappingDigest},
|
||
CreatedAt: finishedAt,
|
||
})
|
||
if versionErr != nil {
|
||
message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error()
|
||
recorder.Stage(ctx, "finalize", "failed", message)
|
||
// 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。
|
||
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
|
||
message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记",
|
||
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
|
||
log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr)
|
||
}
|
||
return errors.New(message)
|
||
}
|
||
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本")
|
||
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
|
||
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
|
||
// 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。
|
||
return fmt.Errorf("作业状态写入失败: %w", advanceErr)
|
||
}
|
||
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
|
||
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID)
|
||
return nil
|
||
case "rolled_back":
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||
return errors.New(outcome.Error)
|
||
case "rollback_failed":
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
|
||
default:
|
||
recovery := outcome.Recovery
|
||
if recovery == "" {
|
||
recovery = "未对生产执行任何写入,无需恢复"
|
||
}
|
||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup",
|
||
outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt)
|
||
return errors.New(outcome.Error)
|
||
}
|
||
}
|
||
|
||
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业:
|
||
// 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用),
|
||
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。
|
||
func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Store, config appconfig.Config) (int, int, error) {
|
||
interrupted, err := store.InterruptedLibraryJobs(ctx)
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
resumed, failed := 0, 0
|
||
for _, job := range interrupted {
|
||
now := time.Now().UTC()
|
||
if job.SnapshotID == "" {
|
||
message := "worker 中断时还没有完成预备份,不能盲写目标"
|
||
_ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message,
|
||
"未对生产执行写入", "", "", now)
|
||
failed++
|
||
continue
|
||
}
|
||
// 原子抢占:多个 worker 同时接管也只有一个能把 running 改成 pending。
|
||
claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now)
|
||
if claimErr != nil {
|
||
log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr)
|
||
continue
|
||
}
|
||
if !claimed {
|
||
continue
|
||
}
|
||
queueType := librarySyncJobType
|
||
if job.Kind == governance.LibraryJobRestore {
|
||
queueType = libraryRestoreJobType
|
||
}
|
||
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
|
||
"target_snapshot_id": job.TargetSnapshotID})
|
||
// 队列作业 ID 与幂等键都按作业固定:重复接管/重复点击只会得到同一个队列作业。
|
||
queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{
|
||
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload,
|
||
AvailableAt: now, MaxAttempts: 1,
|
||
IdempotencyKey: "library.resume.queue:" + job.ID,
|
||
}, now)
|
||
if err != nil || queueJob == nil {
|
||
// 队列作业 ID 是确定性的:并发接管时另一个 worker 已经建好了,
|
||
// 这里按"已存在"处理并继续接管,不去覆盖别人的队列行。
|
||
if err != nil && strings.Contains(err.Error(), "duplicate key") {
|
||
queueID := governance.QueueJobIDFor(job.ID)
|
||
if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil {
|
||
log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr)
|
||
continue
|
||
}
|
||
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil {
|
||
log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr)
|
||
} else {
|
||
resumed++
|
||
}
|
||
continue
|
||
}
|
||
log.Printf("library job %s resume enqueue failed: %v", job.ID, err)
|
||
continue
|
||
}
|
||
if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil {
|
||
log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr)
|
||
continue
|
||
}
|
||
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil {
|
||
log.Printf("library job %s resume attach failed: %v", job.ID, attachErr)
|
||
continue
|
||
}
|
||
resumed++
|
||
log.Printf("library job %s resumed after worker restart (backup %s)", job.ID, job.SnapshotID)
|
||
}
|
||
return resumed, failed, nil
|
||
}
|
||
|
||
// fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。
|
||
type fixedBackupStore struct{ ref libraryrelease.BackupRef }
|
||
|
||
func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
|
||
return s.ref, nil
|
||
}
|
||
|
||
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误:
|
||
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
|
||
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
|
||
now := time.Now().UTC()
|
||
if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "",
|
||
backup.SnapshotID, "", now); err != nil {
|
||
return fmt.Errorf("持久化预备份引用失败: %w", err)
|
||
}
|
||
if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||
ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID),
|
||
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID,
|
||
SourceJobID: r.jobID, CreatedBy: "worker",
|
||
Evidence: map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes,
|
||
"job": r.jobID, "index_mapping_sha256": r.mappingDigest},
|
||
CreatedAt: now,
|
||
}); err != nil {
|
||
return fmt.Errorf("登记预备份版本失败: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func failLibraryJob(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder, jobID, message, stage string) error {
|
||
recorder.Stage(ctx, stage, "failed", message)
|
||
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message,
|
||
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
|
||
return errors.New(message)
|
||
}
|
||
|
||
func kindLabel(kind governance.LibraryJobKind) string {
|
||
if kind == governance.LibraryJobRestore {
|
||
return "恢复"
|
||
}
|
||
return "同步"
|
||
}
|
||
|
||
func digestKey(kind governance.LibraryJobKind) string {
|
||
if kind == governance.LibraryJobRestore {
|
||
return "restore_snapshot_id"
|
||
}
|
||
return "target_snapshot_id"
|
||
}
|