姚章浩
c72b71d78a
build / build (push) Failing after 17s
为什么：父审指出预备份引用持久化失败仍继续写目标、接管依赖启动扫描且漏掉租约未过期、 恢复失败被标成"无需恢复"、版本登记与作业状态分叉。 - BackupPrepared 返回 error，持久化失败即零写入；预备份同时登记 production_prebackup 版本。 - 接管改为按队列租约判定 + 运行期 30s 扫描 + 原子抢占 + 固定幂等键 + attempt 归零。 - 接管场景保留预备份、不再宣称"未写无需恢复"。 - 恢复/重放同一快照重新激活 production_effective（清 superseded_at）。 - finalize 续作只做校验+登记（finalize_only），不重写生产。 - 索引映射基线缺失时采用首次观察值并登记；索引校验加孤儿文档检查（按 knowledge_id）。 真实验收（候选 API :18089 + 真实 PG/ES/MinIO）： A0 首次发布、A1 撤销、B 按版本恢复、C 写阶段 SIGKILL 接管、 D 跨 kind 互斥与多 worker 单次执行、E1 登记失败、E2 finalize 续作、E3 预备份失败零写入。
255 lines
11 KiB
Go
255 lines
11 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"net/http"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
)
|
||
|
||
// 按指定整库版本恢复：与同步分开的作业类型，需要**单独的管理员授权**，
|
||
// 授权摘要绑定"恢复到这个快照"这一件事，不能拿同步授权顶替。
|
||
const libraryRestoreJobType = "library.restore"
|
||
|
||
func libraryRestoreParamsDigest(scope, snapshotID string) (string, error) {
|
||
return governance.LibraryRequestDigest(scope, governance.LibraryJobRestore,
|
||
map[string]any{"restore_snapshot_id": snapshotID})
|
||
}
|
||
|
||
type libraryRestoreRequest struct {
|
||
Scope string `json:"scope"`
|
||
SnapshotID string `json:"snapshot_id"`
|
||
}
|
||
|
||
func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||
if r.Method != http.MethodPost {
|
||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||
return
|
||
}
|
||
identity, ok := sessionFrom(r.Context())
|
||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "按版本恢复仅管理员可发起"})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_unavailable"})
|
||
return
|
||
}
|
||
var request libraryRestoreRequest
|
||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||
return
|
||
}
|
||
if request.Scope != "internal" && request.Scope != "external" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||
return
|
||
}
|
||
if strings.TrimSpace(request.SnapshotID) == "" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
|
||
return
|
||
}
|
||
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||
return
|
||
}
|
||
// 双闸与恢复演练证据同样适用于恢复：恢复也是写生产。
|
||
reasons := make([]string, 0, 4)
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
reasons = append(reasons, "生产写入闸口关闭")
|
||
}
|
||
if !config.ProductionBackupRestoreVerified {
|
||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||
}
|
||
authorizationOK := false
|
||
if storeWithAuth, ok := config.Store.(interface {
|
||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||
}); ok {
|
||
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), request.Scope, request.SnapshotID, digest, time.Now().UTC())
|
||
authorizationOK = authErr == nil && found
|
||
}
|
||
if !authorizationOK {
|
||
reasons = append(reasons, "没有针对该快照的恢复授权")
|
||
}
|
||
if len(reasons) > 0 {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_blocked", "reasons": reasons})
|
||
return
|
||
}
|
||
now := time.Now().UTC()
|
||
jobID := "librestore-" + request.Scope + "-" + now.Format("20060102T150405.000000")
|
||
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
|
||
ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore,
|
||
IdempotencyKey: "library.restore:" + request.Scope + ":" + request.SnapshotID,
|
||
RequestDigest: digest, TargetSnapshotID: request.SnapshotID, Actor: identity.Username,
|
||
}, governance.SyncStages(), now)
|
||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_active", "error": "该目标已有在飞作业"})
|
||
return
|
||
}
|
||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_conflict", "error": "同一恢复请求参数不一致"})
|
||
return
|
||
}
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create restore job"})
|
||
return
|
||
}
|
||
if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) {
|
||
job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{
|
||
ID: jobID + "-retry", Scope: request.Scope, Kind: governance.LibraryJobRestore,
|
||
IdempotencyKey: "library.restore:" + request.Scope + ":" + request.SnapshotID + ":retry:" + now.Format("20060102T150405.000000000"),
|
||
RequestDigest: digest, TargetSnapshotID: request.SnapshotID, Actor: identity.Username,
|
||
}, governance.SyncStages(), now)
|
||
if err != nil || reused {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_retry_conflict", "error": "无法为重试创建新作业"})
|
||
return
|
||
}
|
||
}
|
||
if reused {
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
|
||
return
|
||
}
|
||
payload, err := json.Marshal(map[string]string{
|
||
"library_job_id": job.ID, "scope": request.Scope, "target_snapshot_id": request.SnapshotID,
|
||
})
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build restore payload"})
|
||
return
|
||
}
|
||
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||
ID: governance.QueueJobIDFor(job.ID), Type: libraryRestoreJobType, Payload: payload, AvailableAt: now,
|
||
MaxAttempts: 1, IdempotencyKey: "library.restore.queue:" + job.ID,
|
||
}, now)
|
||
if err != nil || queueJob == nil {
|
||
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败，未开始恢复", now)
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue restore job"})
|
||
return
|
||
}
|
||
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach restore job"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false})
|
||
}
|
||
|
||
// 崩溃续作：失败作业可以按原参数重新排队，生成新作业（旧作业保留为审计记录）。
|
||
func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
|
||
identity, ok := sessionFrom(r.Context())
|
||
// 续作会重新写生产，权限与恢复一致：只有管理员能触发。
|
||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "续作需要管理员权限"})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不可用"})
|
||
return
|
||
}
|
||
sourceID := strings.TrimSpace(r.PathValue("id"))
|
||
source, err := store.LibraryJob(r.Context(), sourceID)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusNotFound, map[string]any{"error": "作业不存在"})
|
||
return
|
||
}
|
||
if source.State != governance.LibraryJobFailed && source.State != governance.LibraryJobCancelled {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "只有失败或已取消的作业可以续作"})
|
||
return
|
||
}
|
||
// 与同步/恢复一致：闸口、恢复演练证据与授权都要在 API 侧 fail-closed，
|
||
// 不能让前台接受一个 worker 必然拒绝的请求。
|
||
reasons := make([]string, 0, 3)
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
reasons = append(reasons, "生产写入闸口关闭")
|
||
}
|
||
if !config.ProductionBackupRestoreVerified {
|
||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||
}
|
||
var digest string
|
||
switch source.Kind {
|
||
case governance.LibraryJobSync:
|
||
digest, _ = librarySyncParamsDigest(source.Scope, source.TargetSnapshotID)
|
||
case governance.LibraryJobRestore:
|
||
digest, _ = libraryRestoreParamsDigest(source.Scope, source.TargetSnapshotID)
|
||
}
|
||
authorizationOK := false
|
||
if storeWithAuth, ok := config.Store.(interface {
|
||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||
}); ok && digest != "" {
|
||
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), source.Scope,
|
||
source.TargetSnapshotID, digest, time.Now().UTC())
|
||
authorizationOK = authErr == nil && found
|
||
}
|
||
if !authorizationOK {
|
||
reasons = append(reasons, "没有针对该快照的有效授权")
|
||
}
|
||
if len(reasons) > 0 {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_resume_blocked", "reasons": reasons})
|
||
return
|
||
}
|
||
|
||
now := time.Now().UTC()
|
||
// 幂等键按源作业固定：重复点击返回同一个续作作业，而不是每次新建。
|
||
jobID := sourceID + "-resume"
|
||
var kind governance.LibraryJobKind
|
||
queueType := ""
|
||
switch source.Kind {
|
||
case governance.LibraryJobSync:
|
||
kind, queueType = governance.LibraryJobSync, librarySyncJobType
|
||
case governance.LibraryJobRestore:
|
||
kind, queueType = governance.LibraryJobRestore, libraryRestoreJobType
|
||
default:
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "封存作业由封存入口重发，不走续作"})
|
||
return
|
||
}
|
||
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
|
||
ID: jobID, Scope: source.Scope, Kind: kind,
|
||
IdempotencyKey: "library.resume:" + sourceID,
|
||
RequestDigest: digest, TargetSnapshotID: source.TargetSnapshotID, Actor: identity.Username,
|
||
}, governance.SyncStages(), now)
|
||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "该目标已有在飞作业"})
|
||
return
|
||
}
|
||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "同一续作请求参数不一致"})
|
||
return
|
||
}
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create resumed job"})
|
||
return
|
||
}
|
||
if reused {
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "resumed_from": sourceID, "reused": true})
|
||
return
|
||
}
|
||
// 写入已完成、只差登记版本的情况：续作只做幂等收尾，不重写生产。
|
||
body := map[string]any{"library_job_id": job.ID, "scope": source.Scope,
|
||
"target_snapshot_id": source.TargetSnapshotID}
|
||
if source.Stage == "finalize" && source.SnapshotID != "" {
|
||
body["finalize_only"] = true
|
||
}
|
||
payload, _ := json.Marshal(body)
|
||
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload, AvailableAt: now,
|
||
MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + job.ID,
|
||
}, now)
|
||
if err != nil || queueJob == nil {
|
||
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败", now)
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue resumed job"})
|
||
return
|
||
}
|
||
// 附不上队列号就等于作业没人跑：必须显式失败并取消，不能静默返回 202。
|
||
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
|
||
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列归属写入失败", now)
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach resumed job"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "resumed_from": sourceID, "reused": false})
|
||
}
|