feat(library): 按版本恢复入口与失败作业续作
build / build (push) Successful in 1m25s

为什么:恢复此前只有"失败自动回滚",没有手动按版本恢复;失败作业也只能重发封存。
- 迁移 000068 允许 kind=restore;POST /library/restores(管理员 + 该快照恢复授权 + 闸口)。
- POST /library/jobs/{id}/resume 对失败/取消的同步/恢复作业按原参数续作。
- worker 统一 runLibraryRelease,同步与恢复共用链路;成功后登记 production_effective 版本。
This commit is contained in:
姚章浩
2026-10-28 16:27:46 +08:00
parent 0b9dbe12cf
commit a2d051f6b3
11 changed files with 548 additions and 91 deletions
+192
View File
@@ -0,0 +1,192 @@
package httpapi
import (
"context"
"encoding/json"
"errors"
"net/http"
"strings"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
// 按指定整库版本恢复:与同步分开的作业类型,需要**单独的管理员授权**,
// 授权摘要绑定"恢复到这个快照"这一件事,不能拿同步授权顶替。
const libraryRestoreJobType = "library.restore"
func libraryRestoreParamsDigest(scope, snapshotID string) (string, error) {
return governance.LibraryRequestDigest(scope, governance.LibraryJobRestore,
map[string]any{"restore_snapshot_id": snapshotID})
}
type libraryRestoreRequest struct {
Scope string `json:"scope"`
SnapshotID string `json:"snapshot_id"`
}
func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
if r.Method != http.MethodPost {
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
return
}
identity, ok := sessionFrom(r.Context())
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
writeJSON(w, http.StatusForbidden, map[string]any{"error": "按版本恢复仅管理员可发起"})
return
}
store, ok := libraryStore(config)
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_unavailable"})
return
}
var request libraryRestoreRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
return
}
if request.Scope != "internal" && request.Scope != "external" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
return
}
if strings.TrimSpace(request.SnapshotID) == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
return
}
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
return
}
// 双闸与恢复演练证据同样适用于恢复:恢复也是写生产。
reasons := make([]string, 0, 4)
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
authorizationOK := false
if storeWithAuth, ok := config.Store.(interface {
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
}); ok {
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), request.Scope, request.SnapshotID, digest, time.Now().UTC())
authorizationOK = authErr == nil && found
}
if !authorizationOK {
reasons = append(reasons, "没有针对该快照的恢复授权")
}
if len(reasons) > 0 {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_blocked", "reasons": reasons})
return
}
now := time.Now().UTC()
jobID := "librestore-" + request.Scope + "-" + now.Format("20060102T150405.000000")
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore,
IdempotencyKey: "library.restore:" + request.Scope + ":" + request.SnapshotID,
RequestDigest: digest, TargetSnapshotID: request.SnapshotID, Actor: identity.Username,
}, governance.SyncStages(), now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_active", "error": "该目标已有在飞作业"})
return
}
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_conflict", "error": "同一恢复请求参数不一致"})
return
}
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create restore job"})
return
}
if reused {
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
return
}
payload, err := json.Marshal(map[string]string{
"library_job_id": job.ID, "scope": request.Scope, "target_snapshot_id": request.SnapshotID,
})
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build restore payload"})
return
}
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: job.ID + "-queue", Type: libraryRestoreJobType, Payload: payload, AvailableAt: now,
MaxAttempts: 1, IdempotencyKey: "library.restore.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始恢复", now)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue restore job"})
return
}
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach restore job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false})
}
// 崩溃续作:失败作业可以按原参数重新排队,生成新作业(旧作业保留为审计记录)。
func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
identity, ok := sessionFrom(r.Context())
if !ok || roleRank(identity.Role) < roleRank(roleEngineer) {
writeJSON(w, http.StatusForbidden, map[string]any{"error": "续作需要知识工程师及以上权限"})
return
}
store, ok := libraryStore(config)
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不可用"})
return
}
sourceID := strings.TrimSpace(r.PathValue("id"))
source, err := store.LibraryJob(r.Context(), sourceID)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]any{"error": "作业不存在"})
return
}
if source.State != governance.LibraryJobFailed && source.State != governance.LibraryJobCancelled {
writeJSON(w, http.StatusConflict, map[string]any{"error": "只有失败或已取消的作业可以续作"})
return
}
now := time.Now().UTC()
jobID := sourceID + "-r" + now.Format("150405")
var kind governance.LibraryJobKind
queueType := ""
switch source.Kind {
case governance.LibraryJobSync:
kind, queueType = governance.LibraryJobSync, librarySyncJobType
case governance.LibraryJobRestore:
kind, queueType = governance.LibraryJobRestore, libraryRestoreJobType
default:
writeJSON(w, http.StatusConflict, map[string]any{"error": "封存作业由封存入口重发,不走续作"})
return
}
digest := source.RequestDigest
job, _, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
ID: jobID, Scope: source.Scope, Kind: kind,
IdempotencyKey: "library.resume:" + sourceID + ":" + now.Format("150405"),
RequestDigest: digest, TargetSnapshotID: source.TargetSnapshotID, Actor: identity.Username,
}, governance.SyncStages(), now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"error": "该目标已有在飞作业"})
return
}
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create resumed job"})
return
}
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": source.Scope,
"target_snapshot_id": source.TargetSnapshotID})
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: job.ID + "-queue", Type: queueType, Payload: payload, AvailableAt: now,
MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败", now)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue resumed job"})
return
}
_ = store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now)
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "resumed_from": sourceID})
}
+87
View File
@@ -0,0 +1,87 @@
package httpapi
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"knowledge-governance/internal/governance"
)
func restoreRequest(body, role string) *http.Request {
request := httptest.NewRequest(http.MethodPost, "/api/v1/library/restores", strings.NewReader(body))
request.Header.Set("Content-Type", "application/json")
ctx := context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: role})
return request.WithContext(ctx)
}
// 恢复是写生产:非管理员直接拒绝,任何授权都不会被创建。
func TestLibraryRestoreRequiresAdmin(t *testing.T) {
store := &fakeLibraryStore{}
recorder := httptest.NewRecorder()
libraryRestoreEnqueue(recorder, restoreRequest(`{"scope":"internal","snapshot_id":"snap-1"}`, roleEngineer), Config{Store: store})
if recorder.Code != http.StatusForbidden {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
if store.createN != 0 {
t.Fatal("non-admin must not create a restore job")
}
}
// 闸口关闭 / 缺恢复演练证据 / 缺该快照授权:逐条拒绝且不建作业。
func TestLibraryRestoreRefusesWithoutGatesAndAuthorization(t *testing.T) {
store := &fakeLibraryStore{}
recorder := httptest.NewRecorder()
libraryRestoreEnqueue(recorder, restoreRequest(`{"scope":"internal","snapshot_id":"snap-1"}`, roleAdmin), Config{Store: store})
if recorder.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
body := recorder.Body.String()
for _, want := range []string{"生产写入闸口关闭", "恢复演练证据", "恢复授权"} {
if !strings.Contains(body, want) {
t.Fatalf("refusal must name %q: %s", want, body)
}
}
if store.createN != 0 {
t.Fatal("must not create a restore job while blocked")
}
}
// 授权必须绑定"恢复到这个快照":同步授权不能顶替。
func TestLibraryRestoreRequiresSnapshotBoundGrant(t *testing.T) {
store := &fakeLibraryStore{}
syncDigest, err := librarySyncParamsDigest("internal", "snap-1")
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
store.grant = &governance.LibraryReleaseAuthorization{ID: "a", Scope: "internal", SnapshotID: "snap-1",
ParamsDigest: syncDigest, ExpiresAt: now.Add(time.Hour), CreatedAt: now}
config := Config{Store: store, ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: true}
recorder := httptest.NewRecorder()
libraryRestoreEnqueue(recorder, restoreRequest(`{"scope":"internal","snapshot_id":"snap-1"}`, roleAdmin), config)
if recorder.Code != http.StatusServiceUnavailable || !strings.Contains(recorder.Body.String(), "恢复授权") {
t.Fatalf("sync grant must not authorize a restore: %d %s", recorder.Code, recorder.Body.String())
}
if store.createN != 0 {
t.Fatal("must not create a restore job with a sync-only grant")
}
}
// 续作只对失败/取消的同步与恢复作业开放;封存作业走封存入口。
func TestLibraryJobResumeRejectsNonFailedJobs(t *testing.T) {
store := &fakeLibraryStore{jobs: []governance.LibraryJob{{ID: "job-1", Scope: "internal",
Kind: governance.LibraryJobSync, State: governance.LibraryJobSucceeded}}}
request := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/job-1/resume", nil)
request.SetPathValue("id", "job-1")
request = request.WithContext(context.WithValue(request.Context(), sessionKey{},
sessionIdentity{Username: "eng", Role: roleEngineer}))
recorder := httptest.NewRecorder()
libraryJobResume(recorder, request, Config{Store: store})
if recorder.Code != http.StatusConflict {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
}
+8
View File
@@ -434,6 +434,14 @@ func NewHandler(config Config) (http.Handler, error) {
mux.HandleFunc("POST /api/v1/library/authorizations/{id}/revoke", func(w http.ResponseWriter, r *http.Request) {
libraryAuthorizationRevoke(w, r, config)
})
// 按指定整库版本恢复(管理员 + 该快照的恢复授权)。
mux.HandleFunc("POST /api/v1/library/restores", func(w http.ResponseWriter, r *http.Request) {
libraryRestoreEnqueue(w, r, config)
})
// 失败/取消作业续作:按原参数重新排队,旧作业保留审计。
mux.HandleFunc("POST /api/v1/library/jobs/{id}/resume", func(w http.ResponseWriter, r *http.Request) {
libraryJobResume(w, r, config)
})
// 生产同步唯一入口:闸口未齐不建作业,返回缺项而不是假 Accepted。
mux.HandleFunc("POST /api/v1/library/syncs", func(w http.ResponseWriter, r *http.Request) {
librarySyncEnqueue(w, r, config)