Files
knowledge-governance/internal/httpapi/library_restore.go
T
姚章浩 98294b9583
build / build (push) Failing after 24s
fix(library): 统一执行身份、终态收尾、准入与数据身份
why:审计 19 项(12 P1 / 7 P2)来自同一批根因,逐点试错会把协议撕成十几份:
  · 意图与执行混为一谈(F01/F15):`library.sync:<scope>:<快照>` 是终身幂等键,
    A→B→A 的第三次发布拿到第一次的历史成功作业,什么都不做;
  · 版本行既是实体又是事件(F02/F09):主键只由 (kind,scope,快照) 决定、写入是
    upsert,第二个作业覆盖 `source_job_id`/`evidence`/`created_at`;
  · 接管没有代次(F13):两个 worker 都能认领同一 pending 作业,失租的旧执行还能
    写业务状态、释放别人的维护锁;
  · 早退与终态没有统一政策(F03/F04/F05/F06):半写窗口的前置检查失败会重新开门、
    服务起不来也照样开入口、journal 终态后收尾不可重建、恢复成功不回写 current;
  · 数据身份与准入各写一套(F07/F08/F14/F17):接管丢了复用语义、全库搬运不改
    location_hash、通用入口可造 library.* 作业、目标模型从没被问过;
  · 存储/容量/UI/CI 判据各自为政(F10/F11/F12/F16/F18/F19)。

what:按批准的一套协议集中实施(详见 handoffs/release-consolidated-implementation/
current-contract.md):
  · operation(一次明确意图)/ attempt(epoch 代次)/ 不可变版本事件 / current 指针;
    治理库迁移 76,只动治理库,不碰 WeKnora Schema/源码/镜像;
  · 所有业务状态、阶段、版本事件、维护锁写入带 epoch 守卫(失租代次写不进去);
    心跳失租立刻取消 handler;
  · 半写窗口早退保持关闭;服务未全部健康绝不开入口;journal 终态可从持久事实重建
    收尾;恢复成功登记 recovery 事件并推进 current;
  · worker 在动维护前核对业务行/kind/scope/队列关联;通用入口拒绝 library.*;
  · 全库搬运重算 location_hash 并与文档级导入共用实现;接管消费写集合的复用决定;
    发布前核目标实时模型契约、容量门禁前置;
  · 存储保护闭包改为全量依赖、类型只认显式登记、保护未知则不出可删候选;
  · 页面核 HTTP 状态与结构、恢复纳入轮询、Wiki 差异覆盖标题/摘要/父级。

how:`make verify` rc=0(16 项必需检查全绿,含 go test 全量与前端 485 例);
`go vet ./...` rc=0;`npx tsc -p tsconfig.json --noEmit` rc=0;逐项负例见各包测试
(F03/F04/F08/F12/F13/F14/F16/F18/F19 都有先红后绿记录)。
未执行:治理库迁移到任何已有库、真实 PG/ES/S3/WeKnora 集成、故障注入、部署。
2026-10-30 16:14:54 +08:00

352 lines
16 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package httpapi
import (
"context"
"encoding/json"
"errors"
"net/http"
"strconv"
"strings"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
// 按指定整库版本恢复:与同步分开的作业类型,需要**单独的管理员授权**,
// 授权摘要绑定"恢复到这个快照"这一件事,不能拿同步授权顶替。
const libraryRestoreJobType = "library.restore"
func libraryRestoreParamsDigest(scope, snapshotID string) (string, error) {
return governance.LibraryRequestDigest(scope, governance.LibraryJobRestore,
map[string]any{"restore_snapshot_id": snapshotID})
}
type libraryRestoreRequest struct {
Scope string `json:"scope"`
SnapshotID string `json:"snapshot_id"`
// OperationKey 是客户端重放键;不传就是一次新意图(同一快照再恢复一次也要真的执行)。
OperationKey string `json:"operation_key"`
}
func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
if r.Method != http.MethodPost {
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
return
}
identity, ok := sessionFrom(r.Context())
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
writeJSON(w, http.StatusForbidden, map[string]any{"error": "按版本恢复仅管理员可发起"})
return
}
store, ok := libraryStore(config)
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_unavailable"})
return
}
var request libraryRestoreRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
return
}
if request.Scope != "internal" && request.Scope != "external" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
return
}
if strings.TrimSpace(request.SnapshotID) == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
return
}
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
writeJSON(w, http.StatusConflict, body)
return
}
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
return
}
// 双闸与恢复演练证据同样适用于恢复:恢复也是写生产。
reasons := make([]string, 0, 4)
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
authorizationOK := false
if storeWithAuth, ok := config.Store.(interface {
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
}); ok {
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), request.Scope, request.SnapshotID, digest, time.Now().UTC())
authorizationOK = authErr == nil && found
}
if !authorizationOK {
reasons = append(reasons, "没有针对该快照的恢复授权")
}
if len(reasons) > 0 {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_blocked", "reasons": reasons})
return
}
now := time.Now().UTC()
// 作业 id 会作为维护窗口的 job 标识落进 freeze journal,而冻结协调器只接受
// [A-Za-z0-9_-]:带 "." 的时间戳会让恢复在开窗口时被拒(真跑报 invalid freeze request),
// 所以用纳秒整数,和同步作业保持一致。
jobID := "librestore-" + request.Scope + "-" + strconv.FormatInt(now.UnixNano(), 10)
intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobRestore,
request.SnapshotID, digest, identity.Username, request.OperationKey, now)
if intentErr != nil {
if isIntentConflict(intentErr) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_intent_conflict",
"error": "同一 operation_key 对应了不同的请求内容:换一个键表示新意图"})
return
}
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register restore intent"})
return
}
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore,
IdempotencyKey: intent.JobKey,
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
Actor: identity.Username,
}, governance.SyncStages(), now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_active", "error": "该目标已有在飞作业"})
return
}
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_conflict", "error": "同一恢复请求参数不一致"})
return
}
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create restore job"})
return
}
if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) {
job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{
ID: jobID + "-retry", Scope: request.Scope, Kind: governance.LibraryJobRestore,
IdempotencyKey: intent.JobKey + ":retry:" + strconv.FormatInt(now.UnixNano(), 10),
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
Actor: identity.Username,
}, governance.SyncStages(), now)
if err != nil || reused {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_retry_conflict", "error": "无法为重试创建新作业"})
return
}
}
if reused {
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
return
}
payload, err := json.Marshal(map[string]string{
"library_job_id": job.ID, "scope": request.Scope, "target_snapshot_id": request.SnapshotID,
})
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build restore payload"})
return
}
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: governance.QueueJobIDFor(job.ID), Type: libraryRestoreJobType, Payload: payload, AvailableAt: now,
MaxAttempts: 1, IdempotencyKey: "library.restore.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始恢复", now)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue restore job"})
return
}
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach restore job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false})
}
// libraryRestoreAvailability 回答"现在能不能把生产恢复到这份快照"。
//
// 为什么需要它:页面原来把"恢复至该版本"按钮写死为 disabled,理由是"未接线"——
// 但接口、授权、恢复作业其实都已经在跑。按钮该由真实条件决定:双闸打开、
// 有这份快照的恢复授权、目标实例没有在维护。
type libraryRestoreAvailabilityView struct {
Allowed bool `json:"allowed"`
Reasons []string `json:"reasons"`
Scope string `json:"scope"`
SnapshotID string `json:"snapshot_id"`
Authorization libraryAuthorizationView `json:"authorization"`
}
func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config Config) {
scope, err := scopeFromQuery(r)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
return
}
snapshotID := strings.TrimSpace(r.URL.Query().Get("snapshot_id"))
if snapshotID == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
return
}
view := libraryRestoreAvailabilityView{Scope: scope, SnapshotID: snapshotID, Reasons: []string{}}
if !config.ProductionReleaseRequestsEnabled {
view.Reasons = append(view.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
}
if !config.ProductionBackupRestoreVerified {
view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
}
digest, err := libraryRestoreParamsDigest(scope, snapshotID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
return
}
grant, found := governance.LibraryReleaseAuthorization{}, false
if storeWithAuth, ok := config.Store.(interface {
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
}); ok {
grant, found, err = storeWithAuth.ActiveLibraryAuthorization(r.Context(), scope, snapshotID, digest, time.Now().UTC())
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"allowed": false,
"reasons": []string{"读取恢复授权失败:" + err.Error()}})
return
}
}
view.Authorization = authorizationViewOf(grant, found)
if !found {
view.Reasons = append(view.Reasons, "没有针对该快照的恢复授权:先签发 restore 授权(绑定同一快照)")
}
if _, held := instanceMaintenanceGuard(r.Context(), config, scope); held {
view.Reasons = append(view.Reasons, "目标实例正在维护中,等当前作业释放后再恢复")
}
view.Allowed = len(view.Reasons) == 0
writeJSON(w, http.StatusOK, view)
}
// 崩溃续作:失败作业可以按原参数重新排队,生成新作业(旧作业保留为审计记录)。
func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
identity, ok := sessionFrom(r.Context())
// 续作会重新写生产,权限与恢复一致:只有管理员能触发。
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
writeJSON(w, http.StatusForbidden, map[string]any{"error": "续作需要管理员权限"})
return
}
store, ok := libraryStore(config)
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不可用"})
return
}
sourceID := strings.TrimSpace(r.PathValue("id"))
source, err := store.LibraryJob(r.Context(), sourceID)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]any{"error": "作业不存在"})
return
}
if source.State != governance.LibraryJobFailed && source.State != governance.LibraryJobCancelled {
writeJSON(w, http.StatusConflict, map[string]any{"error": "只有失败或已取消的作业可以续作"})
return
}
// 与同步/恢复一致:闸口、恢复演练证据与授权都要在 API 侧 fail-closed,
// 不能让前台接受一个 worker 必然拒绝的请求。
reasons := make([]string, 0, 3)
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
var digest string
switch source.Kind {
case governance.LibraryJobSync:
digest, _ = librarySyncParamsDigest(source.Scope, source.TargetSnapshotID)
case governance.LibraryJobRestore:
digest, _ = libraryRestoreParamsDigest(source.Scope, source.TargetSnapshotID)
}
authorizationOK := false
if storeWithAuth, ok := config.Store.(interface {
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
}); ok && digest != "" {
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), source.Scope,
source.TargetSnapshotID, digest, time.Now().UTC())
authorizationOK = authErr == nil && found
}
if !authorizationOK {
reasons = append(reasons, "没有针对该快照的有效授权")
}
if len(reasons) > 0 {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_resume_blocked", "reasons": reasons})
return
}
now := time.Now().UTC()
// 幂等键按源作业固定:重复点击返回同一个续作作业,而不是每次新建。
jobID := sourceID + "-resume"
var kind governance.LibraryJobKind
queueType := ""
switch source.Kind {
case governance.LibraryJobSync:
kind, queueType = governance.LibraryJobSync, librarySyncJobType
case governance.LibraryJobRestore:
kind, queueType = governance.LibraryJobRestore, libraryRestoreJobType
default:
writeJSON(w, http.StatusConflict, map[string]any{"error": "封存作业由封存入口重发,不走续作"})
return
}
// F15:续作是**同一个意图的新执行代次**,不是新意图。沿用原作业的 operation,
// 维护窗口、预备份、写集合的谱系都还挂在同一条意图上;新作业行只是这次执行的载体,
// worker 起手会为它开一个更大的 epoch(旧代次从此不能写)。
operationID := source.OperationID
if operationID == "" {
// 迁移前的老作业没有 operation:按原作业身份补一条(同键同摘要 ⇒ 幂等,
// 不会因为点了两次续作而造出两条意图)。
intent, intentErr := ensureLibraryIntent(r.Context(), store, source.Scope, kind,
source.TargetSnapshotID, digest, identity.Username, "resume:"+sourceID, now)
if intentErr != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register resume intent"})
return
}
operationID = intent.Operation.ID
}
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
ID: jobID, Scope: source.Scope, Kind: kind,
IdempotencyKey: "library.job:" + operationID + ":resume",
RequestDigest: digest, OperationID: operationID,
TargetSnapshotID: source.TargetSnapshotID, Actor: identity.Username,
}, governance.SyncStages(), now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"error": "该目标已有在飞作业"})
return
}
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
writeJSON(w, http.StatusConflict, map[string]any{"error": "同一续作请求参数不一致"})
return
}
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create resumed job"})
return
}
if reused {
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "resumed_from": sourceID, "reused": true})
return
}
// 写入已完成、只差登记版本的情况:续作只做幂等收尾,不重写生产。
body := map[string]any{"library_job_id": job.ID, "scope": source.Scope,
"target_snapshot_id": source.TargetSnapshotID}
if source.Stage == "finalize" && source.SnapshotID != "" {
body["finalize_only"] = true
}
payload, _ := json.Marshal(body)
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload, AvailableAt: now,
MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败", now)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue resumed job"})
return
}
// 附不上队列号就等于作业没人跑:必须显式失败并取消,不能静默返回 202。
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列归属写入失败", now)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach resumed job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "resumed_from": sourceID, "reused": false})
}