Files
knowledge-governance/cmd/worker/library_handlers.go
T
姚章浩 c72b71d78a
build / build (push) Failing after 17s
fix(library): 预备份硬门禁 + 崩溃接管 + 五项入口验收
为什么:父审指出预备份引用持久化失败仍继续写目标、接管依赖启动扫描且漏掉租约未过期、
恢复失败被标成"无需恢复"、版本登记与作业状态分叉。

- BackupPrepared 返回 error,持久化失败即零写入;预备份同时登记 production_prebackup 版本。
- 接管改为按队列租约判定 + 运行期 30s 扫描 + 原子抢占 + 固定幂等键 + attempt 归零。
- 接管场景保留预备份、不再宣称"未写无需恢复"。
- 恢复/重放同一快照重新激活 production_effective(清 superseded_at)。
- finalize 续作只做校验+登记(finalize_only),不重写生产。
- 索引映射基线缺失时采用首次观察值并登记;索引校验加孤儿文档检查(按 knowledge_id)。

真实验收(候选 API :18089 + 真实 PG/ES/MinIO):
A0 首次发布、A1 撤销、B 按版本恢复、C 写阶段 SIGKILL 接管、
D 跨 kind 互斥与多 worker 单次执行、E1 登记失败、E2 finalize 续作、E3 预备份失败零写入。
2026-10-28 19:11:37 +08:00

330 lines
16 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"strings"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
type libraryReleasePayload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
TargetSnapshotID string `json:"target_snapshot_id"`
// FinalizeOnly:只做收尾(校验 + 登记生效版本),不重写目标。
// 用于"写入已完成但版本登记失败"的续作,避免把已生效的生产再写一遍。
FinalizeOnly bool `json:"finalize_only,omitempty"`
}
// runLibraryRelease 执行"闸口 → 授权 → 预备份 → 四类写入 → 校验"的完整链路。
// 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。
func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config appconfig.Config,
job jobs.Job, kind governance.LibraryJobKind) error {
var payload libraryReleasePayload
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" {
return fmt.Errorf("invalid library %s payload", kind)
}
now := time.Now().UTC()
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now)
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope}
// 崩溃接管:作业里已经持久化过预备份快照,就直接复用它继续,
// 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。
previous, err := store.LibraryJob(ctx, payload.LibraryJobID)
resumed := err == nil && previous.SnapshotID != ""
if resumed {
recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID)
}
gateReasons := make([]string, 0, 3)
if !config.ProductionReleaseRequestsEnabled {
gateReasons = append(gateReasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
}
if len(gateReasons) > 0 {
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
recorder.Stage(ctx, "prebackup", "failed", message)
// 接管场景下生产可能已经是半写状态:这时"无需恢复"是错的,
// 必须把既有预备份作为恢复基线留住,并保持目标被占用直到处理完。
recovery := "未对生产执行任何写入,无需恢复"
if resumed {
recovery = "生产可能处于半写状态,必须先按预备份 " + previous.SnapshotID + " 恢复后再重试"
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup", message,
recovery, previous.SnapshotID, "", time.Now().UTC())
return errors.New(message)
}
digest, err := governance.LibraryRequestDigest(payload.Scope, kind, map[string]any{digestKey(kind): payload.TargetSnapshotID})
if err != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "无法计算请求摘要", "prebackup")
}
grant, found, err := store.ActiveLibraryAuthorization(ctx, payload.Scope, payload.TargetSnapshotID, digest, now)
if err != nil || !found {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "没有与目标快照匹配的有效发布授权", "prebackup")
}
// 生产索引映射基线:优先当前生效版本,其次任何一次成功预备份(它也记录了生产实际映射)。
// 只认"生效版本"会在恢复把版本 supersede 掉之后误判"索引被改动"。
knownMapping := ""
if versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50); versionErr == nil {
for _, item := range versions {
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
knownMapping = raw
break
}
}
}
if knownMapping == "" {
for _, item := range versions {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
knownMapping = raw
break
}
}
}
}
target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping)
if wireErr == nil {
if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
recorder.mappingDigest = digest
}
}
if wireErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup")
}
defer target.Close()
if payload.FinalizeOnly {
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: payload.TargetSnapshotID}); verifyErr != nil {
message := "生产与目标快照不一致,不能登记为生效版本:" + verifyErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"生产可能处于半写状态,需按预备份恢复后重试", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
mappingDigest, _ := target.Search().MappingSHA256(ctx)
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective,
SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker",
Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only",
"index_mapping_sha256": mappingDigest},
CreatedAt: time.Now().UTC(),
}); err != nil {
message := "收尾续作仍无法登记生效版本:" + err.Error()
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message,
"预备份 "+previous.SnapshotID+" 保留,可再次续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
return nil
}
var backupSource libraryrelease.BackupStore = backups
if resumed {
backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
SnapshotID: previous.SnapshotID}}
}
request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID,
ParamsDigest: digest, Now: now,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
case "succeeded":
mappingDigest := ""
if digest, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
mappingDigest = digest
}
// 收尾阶段:先登记权威版本,登记成功才允许把作业标成功。
// 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。
recorder.Stage(ctx, "finalize", "running", "登记生产生效版本")
versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID),
Scope: payload.Scope,
Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID,
SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy,
Evidence: map[string]any{"backup_snapshot": outcome.Backup.SnapshotID,
"backup_bytes": outcome.Backup.Bytes, "action": string(kind),
"index_mapping_sha256": mappingDigest},
CreatedAt: finishedAt,
})
if versionErr != nil {
message := "生产写入已完成,但生效版本登记失败:" + versionErr.Error()
recorder.Stage(ctx, "finalize", "failed", message)
// 作业不能报成功:预备份仍在,修好数据库后可用 resume 续作收尾。
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, "预备份 "+outcome.Backup.SnapshotID+" 保留,数据库恢复后可用续作重试登记",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
log.Printf("library %s finalize failure could not be recorded: %v", kind, advanceErr)
}
return errors.New(message)
}
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本")
if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil {
// 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。
return fmt.Errorf("作业状态写入失败: %w", advanceErr)
}
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID)
return nil
case "rolled_back":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
default:
recovery := outcome.Recovery
if recovery == "" {
recovery = "未对生产执行任何写入,无需恢复"
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "prebackup",
outcome.Error, recovery, outcome.Backup.SnapshotID, "", finishedAt)
return errors.New(outcome.Error)
}
}
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业:
// 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用),
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。
func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Store, config appconfig.Config) (int, int, error) {
interrupted, err := store.InterruptedLibraryJobs(ctx)
if err != nil {
return 0, 0, err
}
resumed, failed := 0, 0
for _, job := range interrupted {
now := time.Now().UTC()
if job.SnapshotID == "" {
message := "worker 中断时还没有完成预备份,不能盲写目标"
_ = store.AdvanceLibraryJob(ctx, job.ID, governance.LibraryJobFailed, "prebackup", message,
"未对生产执行写入", "", "", now)
failed++
continue
}
// 原子抢占:多个 worker 同时接管也只有一个能把 running 改成 pending。
claimed, claimErr := store.ClaimLibraryJobForResume(ctx, job.ID, now)
if claimErr != nil {
log.Printf("library job %s cannot be claimed for resume: %v", job.ID, claimErr)
continue
}
if !claimed {
continue
}
queueType := librarySyncJobType
if job.Kind == governance.LibraryJobRestore {
queueType = libraryRestoreJobType
}
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
"target_snapshot_id": job.TargetSnapshotID})
// 队列作业 ID 与幂等键都按作业固定:重复接管/重复点击只会得到同一个队列作业。
queueJob, err := store.Enqueue(ctx, jobs.EnqueueRequest{
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload,
AvailableAt: now, MaxAttempts: 1,
IdempotencyKey: "library.resume.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
// 队列作业 ID 是确定性的:并发接管时另一个 worker 已经建好了,
// 这里按"已存在"处理并继续接管,不去覆盖别人的队列行。
if err != nil && strings.Contains(err.Error(), "duplicate key") {
queueID := governance.QueueJobIDFor(job.ID)
if resetErr := store.ResetQueueJobForResume(ctx, queueID, now); resetErr != nil {
log.Printf("library job %s resume reset existing queue failed: %v", job.ID, resetErr)
continue
}
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueID, now); attachErr != nil {
log.Printf("library job %s resume attach existing queue failed: %v", job.ID, attachErr)
} else {
resumed++
}
continue
}
log.Printf("library job %s resume enqueue failed: %v", job.ID, err)
continue
}
if resetErr := store.ResetQueueJobForResume(ctx, queueJob.ID, now); resetErr != nil {
log.Printf("library job %s resume reset queue failed: %v", job.ID, resetErr)
continue
}
if attachErr := store.AttachLibraryQueueJob(ctx, job.ID, queueJob.ID, now); attachErr != nil {
log.Printf("library job %s resume attach failed: %v", job.ID, attachErr)
continue
}
resumed++
log.Printf("library job %s resumed after worker restart (backup %s)", job.ID, job.SnapshotID)
}
return resumed, failed, nil
}
// fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。
type fixedBackupStore struct{ ref libraryrelease.BackupRef }
func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
return s.ref, nil
}
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误:
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
now := time.Now().UTC()
if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "",
backup.SnapshotID, "", now); err != nil {
return fmt.Errorf("持久化预备份引用失败: %w", err)
}
if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID),
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID,
SourceJobID: r.jobID, CreatedBy: "worker",
Evidence: map[string]any{"backup_snapshot": backup.SnapshotID, "backup_bytes": backup.Bytes,
"job": r.jobID, "index_mapping_sha256": r.mappingDigest},
CreatedAt: now,
}); err != nil {
return fmt.Errorf("登记预备份版本失败: %w", err)
}
return nil
}
func failLibraryJob(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder, jobID, message, stage string) error {
recorder.Stage(ctx, stage, "failed", message)
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message,
"未对生产执行任何写入,无需恢复", "", "", time.Now().UTC())
return errors.New(message)
}
func kindLabel(kind governance.LibraryJobKind) string {
if kind == governance.LibraryJobRestore {
return "恢复"
}
return "同步"
}
func digestKey(kind governance.LibraryJobKind) string {
if kind == governance.LibraryJobRestore {
return "restore_snapshot_id"
}
return "target_snapshot_id"
}