姚章浩
5e1482e349
build / build (push) Successful in 1m40s
根因：生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关， 页面没有任何入口去真跑一次恢复；CLI（librarydrill）又不是 worker 能复用的实现， 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动： - 共享实现 internal/libraryverify（维护隔离→捕获→基线校验→canary→恢复→复核→关窗口）， cmd/librarydrill 改为薄封装，CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业：kind + VerifyStages(capture/verify/canary/restore/finalize)， 与 sync/restore/baseline 共用同一把写互斥；迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications（按 target_id 绑定，换库/索引/桶旧证据自动失效， 同 target+scope+job 幂等）。 - worker：admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现，阶段实时写 library_job_stages，结束后写真实终态；通过写 passed 证据，失败写 failed+原因（不伪成功）； 中断接管按同一作业 id 收掉维护窗口。 - API：POST/GET /api/v1/library/verifications（管理员；闸口关闭/维护中拒绝；返回完整 整库目标、最近证据、失败原因），OpenAPI 同步。 - 闸口：同步/恢复/续作的证据判断改为读当前目标证据；历史 env 仅在证据缺失时兜底， 证据存在时以证据为准。 - 页面：生产保护与恢复面板加「恢复验证」块（完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询），作业类型补 verify 文案。 测试：make verify 16/16 全绿（go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定）； 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
312 lines
14 KiB
Go
312 lines
14 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"knowledge-governance/internal/platform/postgres"
|
||
"net/http"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// 按指定整库版本恢复：与同步分开的作业类型，需要**单独的管理员授权**，
|
||
// 授权摘要绑定"恢复到这个快照"这一件事，不能拿同步授权顶替。
|
||
const libraryRestoreJobType = "library.restore"
|
||
|
||
func libraryRestoreParamsDigest(scope, snapshotID string) (string, error) {
|
||
return governance.LibraryRequestDigest(scope, governance.LibraryJobRestore,
|
||
map[string]any{"restore_snapshot_id": snapshotID})
|
||
}
|
||
|
||
type libraryRestoreRequest struct {
|
||
Scope string `json:"scope"`
|
||
SnapshotID string `json:"snapshot_id"`
|
||
// OperationKey 是客户端重放键；不传就是一次新意图（同一快照再恢复一次也要真的执行）。
|
||
OperationKey string `json:"operation_key"`
|
||
}
|
||
|
||
func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||
if r.Method != http.MethodPost {
|
||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||
return
|
||
}
|
||
identity, ok := sessionFrom(r.Context())
|
||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "按版本恢复仅管理员可发起"})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_unavailable"})
|
||
return
|
||
}
|
||
var request libraryRestoreRequest
|
||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||
return
|
||
}
|
||
if request.Scope != "internal" && request.Scope != "external" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||
return
|
||
}
|
||
if strings.TrimSpace(request.SnapshotID) == "" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
|
||
return
|
||
}
|
||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||
writeJSON(w, http.StatusConflict, body)
|
||
return
|
||
}
|
||
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||
return
|
||
}
|
||
// 双闸与恢复演练证据同样适用于恢复：恢复也是写生产。
|
||
reasons := make([]string, 0, 4)
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
reasons = append(reasons, "生产写入闸口关闭")
|
||
}
|
||
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), request.Scope))...)
|
||
authorizationOK := false
|
||
if storeWithAuth, ok := config.Store.(interface {
|
||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||
}); ok {
|
||
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), request.Scope, request.SnapshotID, digest, time.Now().UTC())
|
||
authorizationOK = authErr == nil && found
|
||
}
|
||
if !authorizationOK {
|
||
reasons = append(reasons, "没有针对该快照的恢复授权")
|
||
}
|
||
if len(reasons) > 0 {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_blocked", "reasons": reasons})
|
||
return
|
||
}
|
||
now := time.Now().UTC()
|
||
// 作业 id 会作为维护窗口的 job 标识落进 freeze journal，而冻结协调器只接受
|
||
// [A-Za-z0-9_-]：带 "." 的时间戳会让恢复在开窗口时被拒（真跑报 invalid freeze request），
|
||
// 所以用纳秒整数，和同步作业保持一致。
|
||
jobID := "librestore-" + request.Scope + "-" + strconv.FormatInt(now.UnixNano(), 10)
|
||
intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobRestore,
|
||
request.SnapshotID, digest, identity.Username, request.OperationKey, now)
|
||
if intentErr != nil {
|
||
if isIntentConflict(intentErr) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_intent_conflict",
|
||
"error": "同一 operation_key 对应了不同的请求内容：换一个键表示新意图"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register restore intent"})
|
||
return
|
||
}
|
||
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
|
||
ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore,
|
||
IdempotencyKey: intent.JobKey,
|
||
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
|
||
Actor: identity.Username,
|
||
}, governance.SyncStages(), config.LibraryHandoffEnabled, now)
|
||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_active", "error": "该目标已有在飞作业"})
|
||
return
|
||
}
|
||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_conflict", "error": "同一恢复请求参数不一致"})
|
||
return
|
||
}
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create restore job"})
|
||
return
|
||
}
|
||
if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) {
|
||
job, reused, err = store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
|
||
ID: jobID + "-retry", Scope: request.Scope, Kind: governance.LibraryJobRestore,
|
||
IdempotencyKey: intent.JobKey + ":retry:" + strconv.FormatInt(now.UnixNano(), 10),
|
||
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
|
||
Actor: identity.Username,
|
||
}, governance.SyncStages(), config.LibraryHandoffEnabled, now)
|
||
if err != nil || reused {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_retry_conflict", "error": "无法为重试创建新作业"})
|
||
return
|
||
}
|
||
}
|
||
if reused {
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false})
|
||
}
|
||
|
||
// libraryRestoreAvailability 回答"现在能不能把生产恢复到这份快照"。
|
||
//
|
||
// 为什么需要它：页面原来把"恢复至该版本"按钮写死为 disabled，理由是"未接线"——
|
||
// 但接口、授权、恢复作业其实都已经在跑。按钮该由真实条件决定：双闸打开、
|
||
// 有这份快照的恢复授权、目标实例没有在维护。
|
||
type libraryRestoreAvailabilityView struct {
|
||
Allowed bool `json:"allowed"`
|
||
Reasons []string `json:"reasons"`
|
||
Scope string `json:"scope"`
|
||
SnapshotID string `json:"snapshot_id"`
|
||
Authorization libraryAuthorizationView `json:"authorization"`
|
||
}
|
||
|
||
func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config Config) {
|
||
scope, err := scopeFromQuery(r)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||
return
|
||
}
|
||
snapshotID := strings.TrimSpace(r.URL.Query().Get("snapshot_id"))
|
||
if snapshotID == "" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
|
||
return
|
||
}
|
||
view := libraryRestoreAvailabilityView{Scope: scope, SnapshotID: snapshotID, Reasons: []string{}}
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
view.Reasons = append(view.Reasons, "生产写入闸口关闭（PRODUCTION_RELEASE_REQUESTS_ENABLED=false）")
|
||
}
|
||
view.Reasons = append(view.Reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), scope))...)
|
||
digest, err := libraryRestoreParamsDigest(scope, snapshotID)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||
return
|
||
}
|
||
grant, found := governance.LibraryReleaseAuthorization{}, false
|
||
if storeWithAuth, ok := config.Store.(interface {
|
||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||
}); ok {
|
||
grant, found, err = storeWithAuth.ActiveLibraryAuthorization(r.Context(), scope, snapshotID, digest, time.Now().UTC())
|
||
if err != nil {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"allowed": false,
|
||
"reasons": []string{"读取恢复授权失败：" + err.Error()}})
|
||
return
|
||
}
|
||
}
|
||
view.Authorization = authorizationViewOf(grant, found)
|
||
if !found {
|
||
view.Reasons = append(view.Reasons, "没有针对该快照的恢复授权：先签发 restore 授权（绑定同一快照）")
|
||
}
|
||
if _, held := instanceMaintenanceGuard(r.Context(), config, scope); held {
|
||
view.Reasons = append(view.Reasons, "目标实例正在维护中，等当前作业释放后再恢复")
|
||
}
|
||
view.Allowed = len(view.Reasons) == 0
|
||
writeJSON(w, http.StatusOK, view)
|
||
}
|
||
|
||
// 崩溃续作：失败作业可以按原参数重新排队，生成新作业（旧作业保留为审计记录）。
|
||
func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
|
||
identity, ok := sessionFrom(r.Context())
|
||
// 续作会重新写生产，权限与恢复一致：只有管理员能触发。
|
||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "续作需要管理员权限"})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不可用"})
|
||
return
|
||
}
|
||
sourceID := strings.TrimSpace(r.PathValue("id"))
|
||
source, err := store.LibraryJob(r.Context(), sourceID)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusNotFound, map[string]any{"error": "作业不存在"})
|
||
return
|
||
}
|
||
if source.State != governance.LibraryJobFailed && source.State != governance.LibraryJobCancelled {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "只有失败或已取消的作业可以续作"})
|
||
return
|
||
}
|
||
// 与同步/恢复一致：闸口、恢复演练证据与授权都要在 API 侧 fail-closed，
|
||
// 不能让前台接受一个 worker 必然拒绝的请求。
|
||
reasons := make([]string, 0, 3)
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
reasons = append(reasons, "生产写入闸口关闭")
|
||
}
|
||
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), source.Scope))...)
|
||
var digest string
|
||
switch source.Kind {
|
||
case governance.LibraryJobSync:
|
||
digest, _ = librarySyncParamsDigest(source.Scope, source.TargetSnapshotID)
|
||
case governance.LibraryJobRestore:
|
||
digest, _ = libraryRestoreParamsDigest(source.Scope, source.TargetSnapshotID)
|
||
}
|
||
authorizationOK := false
|
||
if storeWithAuth, ok := config.Store.(interface {
|
||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||
}); ok && digest != "" {
|
||
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), source.Scope,
|
||
source.TargetSnapshotID, digest, time.Now().UTC())
|
||
authorizationOK = authErr == nil && found
|
||
}
|
||
if !authorizationOK {
|
||
reasons = append(reasons, "没有针对该快照的有效授权")
|
||
}
|
||
if len(reasons) > 0 {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_resume_blocked", "reasons": reasons})
|
||
return
|
||
}
|
||
|
||
now := time.Now().UTC()
|
||
queueType := ""
|
||
switch source.Kind {
|
||
case governance.LibraryJobSync:
|
||
queueType = librarySyncJobType
|
||
case governance.LibraryJobRestore:
|
||
queueType = libraryRestoreJobType
|
||
default:
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "封存作业由封存入口重发，不走续作"})
|
||
return
|
||
}
|
||
// R3：续作沿用**原业务作业**，只新增执行代次。
|
||
//
|
||
// 维护窗口 owner_job_id、freeze journal 文件名、预备份与写集合的身份全部按业务作业
|
||
// id 落盘。另建 `-resume` 作业只能继承 operation，其它所有者仍指向旧作业：
|
||
// AcquireInstanceMaintenance 会拒绝它、journal 与预备份也查不到——那样"续作"只是
|
||
// 一个漂亮的新行，接不上原窗口。原作业 + 新 attempt 才是批准协议的形状。
|
||
// 阶段/状态/执行权与载荷全部在存储层的事务里校验并构造：handler 手里的 source 只用于
|
||
// 授权与闸口判断，不参与续作决策，避免用过期快照复活一个已经变化的作业。
|
||
resumer, ok := config.Store.(interface {
|
||
ResumeLibraryJob(ctx context.Context, jobID, queueType string, now time.Time) (governance.LibraryJob, bool, error)
|
||
})
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不支持续作"})
|
||
return
|
||
}
|
||
prepared, resumed, err := resumer.ResumeLibraryJob(r.Context(), sourceID, queueType, now)
|
||
switch {
|
||
case errors.Is(err, postgres.ErrLibraryResumeBusy):
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_busy",
|
||
"error": "该作业已有执行在跑（租约仍有效），等它结束再续作"})
|
||
return
|
||
case errors.Is(err, postgres.ErrLibraryResumeRequiresRecover):
|
||
// 窗口是否仍被持有由维护记录决定：已释放的回滚作业说"窗口已释放"，不能一律宣称仍持有。
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_requires_recover",
|
||
"error": "作业停在回滚阶段，请走维护恢复而不是续作",
|
||
"maintenance_held": maintenanceHeldFor(r.Context(), config, source)})
|
||
return
|
||
case err != nil:
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot resume job"})
|
||
return
|
||
case !resumed:
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_resume_state_changed",
|
||
"error": "作业已不在可续作状态（可能已被接管或完成）"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": prepared, "resumed_from": sourceID,
|
||
"resumed": true, "next_attempt_epoch": prepared.AttemptEpoch + 1})
|
||
}
|
||
|
||
// maintenanceHeldFor 只回答"这个作业现在是否仍持有维护窗口"，用于给回滚作业的拒绝文案定性。
|
||
func maintenanceHeldFor(ctx context.Context, config Config, job governance.LibraryJob) bool {
|
||
reader, ok := config.Store.(interface {
|
||
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgres.LibraryMaintenance, error)
|
||
})
|
||
if !ok {
|
||
return false
|
||
}
|
||
m, err := reader.LibraryMaintenanceOf(ctx, job.Scope)
|
||
return err == nil && m != nil && m.OwnerJobID == job.ID && m.ReleasedAt == nil
|
||
}
|