Files
knowledge-governance/internal/httpapi/library_restore.go
T
姚章浩 a7d643635a
build / build (push) Successful in 1m27s
fix(library): 守卫按行取作业身份、同作业多代次、原作业续作与执行域准入
R1 维护/发布守卫硬编码 j.id=$1:维护类入口的 $1 是 scope/instance,作业号在 $2,
导致 update/release/ingress 六个入口正常路径全部匹配不到作业。改为按被更新行自己的
owner_job_id 生成条件,并同时要求该代次在 library_attempts 里仍活跃。

R2 认领只改 jobs.attempt_epoch 不改 attempts.epoch,第二次接管即错位;唯一索引
(job_id) 与"每次接管新 attempt 且历史不可变"矛盾。改为同作业多代次:锁作业行与
operation 行、同 owner 幂等续租、活跃未过期者让路、过期/本作业旧代次标 abandoned、
新代次 epoch=max+1,并清同 operation 其它作业行租约。迟到 write/renew/finish 一律
ErrLibraryEpochStale(finish 补了"只能在活跃态进来一次")。

R3 续作不再新建 -resume 作业:PrepareLibraryJobResume 让原业务作业回 pending 并显式
结束上一代,维护窗口/journal/预备份/写集合仍按原作业 id 归属,代次由 worker 起手递增。

R4 执行域准入改为注册 handler 前读事实:目标实况(KB/租户/schema/索引)、旧 release
路由绑定的 KB、未释放维护窗口持有者;未知或重叠即拒绝执行(不做维护动作、不停既有服务)。

同时自纠 5 项设计偏离:current 次序键取意图创建时间(旧意图迟到事实被拒)、首次锚点
改事务 FOR UPDATE + 回读且 NULL≠0、版本事件冲突只做加法且 xmax=0 才算新事实、
恢复作为新 kind=recover 意图推进 current、隔离测试加测试库护栏并清理残留 schema。

测试:真实 PG(kg-pgtest-writeset,一用例一 schema)新增 8 条集成用例覆盖上述正负例;
go test ./... 全绿(50 包)、go vet/build rc=0、web tsc + vitest 485 通过、make verify 16 项通过。
2026-10-30 17:20:40 +08:00

333 lines
15 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package httpapi
import (
"context"
"encoding/json"
"errors"
"net/http"
"strconv"
"strings"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
// 按指定整库版本恢复:与同步分开的作业类型,需要**单独的管理员授权**,
// 授权摘要绑定"恢复到这个快照"这一件事,不能拿同步授权顶替。
const libraryRestoreJobType = "library.restore"
func libraryRestoreParamsDigest(scope, snapshotID string) (string, error) {
return governance.LibraryRequestDigest(scope, governance.LibraryJobRestore,
map[string]any{"restore_snapshot_id": snapshotID})
}
type libraryRestoreRequest struct {
Scope string `json:"scope"`
SnapshotID string `json:"snapshot_id"`
// OperationKey 是客户端重放键;不传就是一次新意图(同一快照再恢复一次也要真的执行)。
OperationKey string `json:"operation_key"`
}
func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
if r.Method != http.MethodPost {
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
return
}
identity, ok := sessionFrom(r.Context())
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
writeJSON(w, http.StatusForbidden, map[string]any{"error": "按版本恢复仅管理员可发起"})
return
}
store, ok := libraryStore(config)
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_unavailable"})
return
}
var request libraryRestoreRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
return
}
if request.Scope != "internal" && request.Scope != "external" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
return
}
if strings.TrimSpace(request.SnapshotID) == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
return
}
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
writeJSON(w, http.StatusConflict, body)
return
}
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
return
}
// 双闸与恢复演练证据同样适用于恢复:恢复也是写生产。
reasons := make([]string, 0, 4)
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
authorizationOK := false
if storeWithAuth, ok := config.Store.(interface {
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
}); ok {
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), request.Scope, request.SnapshotID, digest, time.Now().UTC())
authorizationOK = authErr == nil && found
}
if !authorizationOK {
reasons = append(reasons, "没有针对该快照的恢复授权")
}
if len(reasons) > 0 {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_blocked", "reasons": reasons})
return
}
now := time.Now().UTC()
// 作业 id 会作为维护窗口的 job 标识落进 freeze journal,而冻结协调器只接受
// [A-Za-z0-9_-]:带 "." 的时间戳会让恢复在开窗口时被拒(真跑报 invalid freeze request),
// 所以用纳秒整数,和同步作业保持一致。
jobID := "librestore-" + request.Scope + "-" + strconv.FormatInt(now.UnixNano(), 10)
intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobRestore,
request.SnapshotID, digest, identity.Username, request.OperationKey, now)
if intentErr != nil {
if isIntentConflict(intentErr) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_intent_conflict",
"error": "同一 operation_key 对应了不同的请求内容:换一个键表示新意图"})
return
}
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register restore intent"})
return
}
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore,
IdempotencyKey: intent.JobKey,
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
Actor: identity.Username,
}, governance.SyncStages(), now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_active", "error": "该目标已有在飞作业"})
return
}
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_conflict", "error": "同一恢复请求参数不一致"})
return
}
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create restore job"})
return
}
if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) {
job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{
ID: jobID + "-retry", Scope: request.Scope, Kind: governance.LibraryJobRestore,
IdempotencyKey: intent.JobKey + ":retry:" + strconv.FormatInt(now.UnixNano(), 10),
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
Actor: identity.Username,
}, governance.SyncStages(), now)
if err != nil || reused {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_retry_conflict", "error": "无法为重试创建新作业"})
return
}
}
if reused {
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
return
}
payload, err := json.Marshal(map[string]string{
"library_job_id": job.ID, "scope": request.Scope, "target_snapshot_id": request.SnapshotID,
})
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build restore payload"})
return
}
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: governance.QueueJobIDFor(job.ID), Type: libraryRestoreJobType, Payload: payload, AvailableAt: now,
MaxAttempts: 1, IdempotencyKey: "library.restore.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始恢复", now)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue restore job"})
return
}
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach restore job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false})
}
// libraryRestoreAvailability 回答"现在能不能把生产恢复到这份快照"。
//
// 为什么需要它:页面原来把"恢复至该版本"按钮写死为 disabled,理由是"未接线"——
// 但接口、授权、恢复作业其实都已经在跑。按钮该由真实条件决定:双闸打开、
// 有这份快照的恢复授权、目标实例没有在维护。
type libraryRestoreAvailabilityView struct {
Allowed bool `json:"allowed"`
Reasons []string `json:"reasons"`
Scope string `json:"scope"`
SnapshotID string `json:"snapshot_id"`
Authorization libraryAuthorizationView `json:"authorization"`
}
func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config Config) {
scope, err := scopeFromQuery(r)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
return
}
snapshotID := strings.TrimSpace(r.URL.Query().Get("snapshot_id"))
if snapshotID == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
return
}
view := libraryRestoreAvailabilityView{Scope: scope, SnapshotID: snapshotID, Reasons: []string{}}
if !config.ProductionReleaseRequestsEnabled {
view.Reasons = append(view.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
}
if !config.ProductionBackupRestoreVerified {
view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
}
digest, err := libraryRestoreParamsDigest(scope, snapshotID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
return
}
grant, found := governance.LibraryReleaseAuthorization{}, false
if storeWithAuth, ok := config.Store.(interface {
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
}); ok {
grant, found, err = storeWithAuth.ActiveLibraryAuthorization(r.Context(), scope, snapshotID, digest, time.Now().UTC())
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"allowed": false,
"reasons": []string{"读取恢复授权失败:" + err.Error()}})
return
}
}
view.Authorization = authorizationViewOf(grant, found)
if !found {
view.Reasons = append(view.Reasons, "没有针对该快照的恢复授权:先签发 restore 授权(绑定同一快照)")
}
if _, held := instanceMaintenanceGuard(r.Context(), config, scope); held {
view.Reasons = append(view.Reasons, "目标实例正在维护中,等当前作业释放后再恢复")
}
view.Allowed = len(view.Reasons) == 0
writeJSON(w, http.StatusOK, view)
}
// 崩溃续作:失败作业可以按原参数重新排队,生成新作业(旧作业保留为审计记录)。
func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
identity, ok := sessionFrom(r.Context())
// 续作会重新写生产,权限与恢复一致:只有管理员能触发。
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
writeJSON(w, http.StatusForbidden, map[string]any{"error": "续作需要管理员权限"})
return
}
store, ok := libraryStore(config)
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不可用"})
return
}
sourceID := strings.TrimSpace(r.PathValue("id"))
source, err := store.LibraryJob(r.Context(), sourceID)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]any{"error": "作业不存在"})
return
}
if source.State != governance.LibraryJobFailed && source.State != governance.LibraryJobCancelled {
writeJSON(w, http.StatusConflict, map[string]any{"error": "只有失败或已取消的作业可以续作"})
return
}
// 与同步/恢复一致:闸口、恢复演练证据与授权都要在 API 侧 fail-closed,
// 不能让前台接受一个 worker 必然拒绝的请求。
reasons := make([]string, 0, 3)
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
var digest string
switch source.Kind {
case governance.LibraryJobSync:
digest, _ = librarySyncParamsDigest(source.Scope, source.TargetSnapshotID)
case governance.LibraryJobRestore:
digest, _ = libraryRestoreParamsDigest(source.Scope, source.TargetSnapshotID)
}
authorizationOK := false
if storeWithAuth, ok := config.Store.(interface {
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
}); ok && digest != "" {
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), source.Scope,
source.TargetSnapshotID, digest, time.Now().UTC())
authorizationOK = authErr == nil && found
}
if !authorizationOK {
reasons = append(reasons, "没有针对该快照的有效授权")
}
if len(reasons) > 0 {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_resume_blocked", "reasons": reasons})
return
}
now := time.Now().UTC()
queueType := ""
switch source.Kind {
case governance.LibraryJobSync:
queueType = librarySyncJobType
case governance.LibraryJobRestore:
queueType = libraryRestoreJobType
default:
writeJSON(w, http.StatusConflict, map[string]any{"error": "封存作业由封存入口重发,不走续作"})
return
}
// R3:续作沿用**原业务作业**,只新增执行代次。
//
// 维护窗口 owner_job_id、freeze journal 文件名、预备份与写集合的身份全部按业务作业
// id 落盘。另建 `-resume` 作业只能继承 operation,其它所有者仍指向旧作业:
// AcquireInstanceMaintenance 会拒绝它、journal 与预备份也查不到——那样"续作"只是
// 一个漂亮的新行,接不上原窗口。原作业 + 新 attempt 才是批准协议的形状。
prepared, resumed, err := store.PrepareLibraryJobResume(r.Context(), sourceID, now)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot prepare resumed job"})
return
}
if !resumed {
// 状态在检查之后被别的执行者推进(已接管/已完成):不许再塞一次。
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_state_changed",
"error": "作业已不在可续作状态(可能已被接管或完成)"})
return
}
// 写入已完成、只差登记版本的情况:续作只做幂等收尾,不重写生产。
body := map[string]any{"library_job_id": prepared.ID, "scope": source.Scope,
"target_snapshot_id": source.TargetSnapshotID}
if source.Stage == "finalize" && source.SnapshotID != "" {
body["finalize_only"] = true
}
payload, _ := json.Marshal(body)
queueID := governance.QueueJobIDFor(prepared.ID)
if _, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: queueID, Type: queueType, Payload: payload, AvailableAt: now,
MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + prepared.ID,
}, now); err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue resumed job"})
return
}
// 队列行可能还是上一次的终态:重置为 pending。仍有效的租约 ⇒ 有执行在跑,让路。
if resetErr := store.ResetQueueJobForResume(r.Context(), queueID, now); resetErr != nil {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_busy",
"error": "该作业已有执行在跑(租约仍有效),等它结束再续作", "detail": errString(resetErr)})
return
}
if attachErr := store.AttachLibraryQueueJob(r.Context(), prepared.ID, queueID, now); attachErr != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach resumed job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": prepared, "resumed_from": sourceID,
"resumed": true, "next_attempt_epoch": prepared.AttemptEpoch + 1})
}