98294b9583
build / build (push) Failing after 24s
why:审计 19 项(12 P1 / 7 P2)来自同一批根因,逐点试错会把协议撕成十几份:
· 意图与执行混为一谈(F01/F15):`library.sync:<scope>:<快照>` 是终身幂等键,
A→B→A 的第三次发布拿到第一次的历史成功作业,什么都不做;
· 版本行既是实体又是事件(F02/F09):主键只由 (kind,scope,快照) 决定、写入是
upsert,第二个作业覆盖 `source_job_id`/`evidence`/`created_at`;
· 接管没有代次(F13):两个 worker 都能认领同一 pending 作业,失租的旧执行还能
写业务状态、释放别人的维护锁;
· 早退与终态没有统一政策(F03/F04/F05/F06):半写窗口的前置检查失败会重新开门、
服务起不来也照样开入口、journal 终态后收尾不可重建、恢复成功不回写 current;
· 数据身份与准入各写一套(F07/F08/F14/F17):接管丢了复用语义、全库搬运不改
location_hash、通用入口可造 library.* 作业、目标模型从没被问过;
· 存储/容量/UI/CI 判据各自为政(F10/F11/F12/F16/F18/F19)。
what:按批准的一套协议集中实施(详见 handoffs/release-consolidated-implementation/
current-contract.md):
· operation(一次明确意图)/ attempt(epoch 代次)/ 不可变版本事件 / current 指针;
治理库迁移 76,只动治理库,不碰 WeKnora Schema/源码/镜像;
· 所有业务状态、阶段、版本事件、维护锁写入带 epoch 守卫(失租代次写不进去);
心跳失租立刻取消 handler;
· 半写窗口早退保持关闭;服务未全部健康绝不开入口;journal 终态可从持久事实重建
收尾;恢复成功登记 recovery 事件并推进 current;
· worker 在动维护前核对业务行/kind/scope/队列关联;通用入口拒绝 library.*;
· 全库搬运重算 location_hash 并与文档级导入共用实现;接管消费写集合的复用决定;
发布前核目标实时模型契约、容量门禁前置;
· 存储保护闭包改为全量依赖、类型只认显式登记、保护未知则不出可删候选;
· 页面核 HTTP 状态与结构、恢复纳入轮询、Wiki 差异覆盖标题/摘要/父级。
how:`make verify` rc=0(16 项必需检查全绿,含 go test 全量与前端 485 例);
`go vet ./...` rc=0;`npx tsc -p tsconfig.json --noEmit` rc=0;逐项负例见各包测试
(F03/F04/F08/F12/F13/F14/F16/F18/F19 都有先红后绿记录)。
未执行:治理库迁移到任何已有库、真实 PG/ES/S3/WeKnora 集成、故障注入、部署。
352 lines
16 KiB
Go
352 lines
16 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"net/http"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
)
|
||
|
||
// 按指定整库版本恢复:与同步分开的作业类型,需要**单独的管理员授权**,
|
||
// 授权摘要绑定"恢复到这个快照"这一件事,不能拿同步授权顶替。
|
||
const libraryRestoreJobType = "library.restore"
|
||
|
||
func libraryRestoreParamsDigest(scope, snapshotID string) (string, error) {
|
||
return governance.LibraryRequestDigest(scope, governance.LibraryJobRestore,
|
||
map[string]any{"restore_snapshot_id": snapshotID})
|
||
}
|
||
|
||
type libraryRestoreRequest struct {
|
||
Scope string `json:"scope"`
|
||
SnapshotID string `json:"snapshot_id"`
|
||
// OperationKey 是客户端重放键;不传就是一次新意图(同一快照再恢复一次也要真的执行)。
|
||
OperationKey string `json:"operation_key"`
|
||
}
|
||
|
||
func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||
if r.Method != http.MethodPost {
|
||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||
return
|
||
}
|
||
identity, ok := sessionFrom(r.Context())
|
||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "按版本恢复仅管理员可发起"})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_unavailable"})
|
||
return
|
||
}
|
||
var request libraryRestoreRequest
|
||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||
return
|
||
}
|
||
if request.Scope != "internal" && request.Scope != "external" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||
return
|
||
}
|
||
if strings.TrimSpace(request.SnapshotID) == "" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
|
||
return
|
||
}
|
||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||
writeJSON(w, http.StatusConflict, body)
|
||
return
|
||
}
|
||
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||
return
|
||
}
|
||
// 双闸与恢复演练证据同样适用于恢复:恢复也是写生产。
|
||
reasons := make([]string, 0, 4)
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
reasons = append(reasons, "生产写入闸口关闭")
|
||
}
|
||
if !config.ProductionBackupRestoreVerified {
|
||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||
}
|
||
authorizationOK := false
|
||
if storeWithAuth, ok := config.Store.(interface {
|
||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||
}); ok {
|
||
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), request.Scope, request.SnapshotID, digest, time.Now().UTC())
|
||
authorizationOK = authErr == nil && found
|
||
}
|
||
if !authorizationOK {
|
||
reasons = append(reasons, "没有针对该快照的恢复授权")
|
||
}
|
||
if len(reasons) > 0 {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_restore_blocked", "reasons": reasons})
|
||
return
|
||
}
|
||
now := time.Now().UTC()
|
||
// 作业 id 会作为维护窗口的 job 标识落进 freeze journal,而冻结协调器只接受
|
||
// [A-Za-z0-9_-]:带 "." 的时间戳会让恢复在开窗口时被拒(真跑报 invalid freeze request),
|
||
// 所以用纳秒整数,和同步作业保持一致。
|
||
jobID := "librestore-" + request.Scope + "-" + strconv.FormatInt(now.UnixNano(), 10)
|
||
intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobRestore,
|
||
request.SnapshotID, digest, identity.Username, request.OperationKey, now)
|
||
if intentErr != nil {
|
||
if isIntentConflict(intentErr) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_intent_conflict",
|
||
"error": "同一 operation_key 对应了不同的请求内容:换一个键表示新意图"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register restore intent"})
|
||
return
|
||
}
|
||
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
|
||
ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore,
|
||
IdempotencyKey: intent.JobKey,
|
||
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
|
||
Actor: identity.Username,
|
||
}, governance.SyncStages(), now)
|
||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_active", "error": "该目标已有在飞作业"})
|
||
return
|
||
}
|
||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_conflict", "error": "同一恢复请求参数不一致"})
|
||
return
|
||
}
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create restore job"})
|
||
return
|
||
}
|
||
if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) {
|
||
job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{
|
||
ID: jobID + "-retry", Scope: request.Scope, Kind: governance.LibraryJobRestore,
|
||
IdempotencyKey: intent.JobKey + ":retry:" + strconv.FormatInt(now.UnixNano(), 10),
|
||
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
|
||
Actor: identity.Username,
|
||
}, governance.SyncStages(), now)
|
||
if err != nil || reused {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_retry_conflict", "error": "无法为重试创建新作业"})
|
||
return
|
||
}
|
||
}
|
||
if reused {
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
|
||
return
|
||
}
|
||
payload, err := json.Marshal(map[string]string{
|
||
"library_job_id": job.ID, "scope": request.Scope, "target_snapshot_id": request.SnapshotID,
|
||
})
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build restore payload"})
|
||
return
|
||
}
|
||
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||
ID: governance.QueueJobIDFor(job.ID), Type: libraryRestoreJobType, Payload: payload, AvailableAt: now,
|
||
MaxAttempts: 1, IdempotencyKey: "library.restore.queue:" + job.ID,
|
||
}, now)
|
||
if err != nil || queueJob == nil {
|
||
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始恢复", now)
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue restore job"})
|
||
return
|
||
}
|
||
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach restore job"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false})
|
||
}
|
||
|
||
// libraryRestoreAvailability 回答"现在能不能把生产恢复到这份快照"。
|
||
//
|
||
// 为什么需要它:页面原来把"恢复至该版本"按钮写死为 disabled,理由是"未接线"——
|
||
// 但接口、授权、恢复作业其实都已经在跑。按钮该由真实条件决定:双闸打开、
|
||
// 有这份快照的恢复授权、目标实例没有在维护。
|
||
type libraryRestoreAvailabilityView struct {
|
||
Allowed bool `json:"allowed"`
|
||
Reasons []string `json:"reasons"`
|
||
Scope string `json:"scope"`
|
||
SnapshotID string `json:"snapshot_id"`
|
||
Authorization libraryAuthorizationView `json:"authorization"`
|
||
}
|
||
|
||
func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config Config) {
|
||
scope, err := scopeFromQuery(r)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||
return
|
||
}
|
||
snapshotID := strings.TrimSpace(r.URL.Query().Get("snapshot_id"))
|
||
if snapshotID == "" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
|
||
return
|
||
}
|
||
view := libraryRestoreAvailabilityView{Scope: scope, SnapshotID: snapshotID, Reasons: []string{}}
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
view.Reasons = append(view.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
|
||
}
|
||
if !config.ProductionBackupRestoreVerified {
|
||
view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
|
||
}
|
||
digest, err := libraryRestoreParamsDigest(scope, snapshotID)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||
return
|
||
}
|
||
grant, found := governance.LibraryReleaseAuthorization{}, false
|
||
if storeWithAuth, ok := config.Store.(interface {
|
||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||
}); ok {
|
||
grant, found, err = storeWithAuth.ActiveLibraryAuthorization(r.Context(), scope, snapshotID, digest, time.Now().UTC())
|
||
if err != nil {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"allowed": false,
|
||
"reasons": []string{"读取恢复授权失败:" + err.Error()}})
|
||
return
|
||
}
|
||
}
|
||
view.Authorization = authorizationViewOf(grant, found)
|
||
if !found {
|
||
view.Reasons = append(view.Reasons, "没有针对该快照的恢复授权:先签发 restore 授权(绑定同一快照)")
|
||
}
|
||
if _, held := instanceMaintenanceGuard(r.Context(), config, scope); held {
|
||
view.Reasons = append(view.Reasons, "目标实例正在维护中,等当前作业释放后再恢复")
|
||
}
|
||
view.Allowed = len(view.Reasons) == 0
|
||
writeJSON(w, http.StatusOK, view)
|
||
}
|
||
|
||
// 崩溃续作:失败作业可以按原参数重新排队,生成新作业(旧作业保留为审计记录)。
|
||
func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
|
||
identity, ok := sessionFrom(r.Context())
|
||
// 续作会重新写生产,权限与恢复一致:只有管理员能触发。
|
||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "续作需要管理员权限"})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "作业存储不可用"})
|
||
return
|
||
}
|
||
sourceID := strings.TrimSpace(r.PathValue("id"))
|
||
source, err := store.LibraryJob(r.Context(), sourceID)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusNotFound, map[string]any{"error": "作业不存在"})
|
||
return
|
||
}
|
||
if source.State != governance.LibraryJobFailed && source.State != governance.LibraryJobCancelled {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "只有失败或已取消的作业可以续作"})
|
||
return
|
||
}
|
||
// 与同步/恢复一致:闸口、恢复演练证据与授权都要在 API 侧 fail-closed,
|
||
// 不能让前台接受一个 worker 必然拒绝的请求。
|
||
reasons := make([]string, 0, 3)
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
reasons = append(reasons, "生产写入闸口关闭")
|
||
}
|
||
if !config.ProductionBackupRestoreVerified {
|
||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||
}
|
||
var digest string
|
||
switch source.Kind {
|
||
case governance.LibraryJobSync:
|
||
digest, _ = librarySyncParamsDigest(source.Scope, source.TargetSnapshotID)
|
||
case governance.LibraryJobRestore:
|
||
digest, _ = libraryRestoreParamsDigest(source.Scope, source.TargetSnapshotID)
|
||
}
|
||
authorizationOK := false
|
||
if storeWithAuth, ok := config.Store.(interface {
|
||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||
}); ok && digest != "" {
|
||
_, found, authErr := storeWithAuth.ActiveLibraryAuthorization(r.Context(), source.Scope,
|
||
source.TargetSnapshotID, digest, time.Now().UTC())
|
||
authorizationOK = authErr == nil && found
|
||
}
|
||
if !authorizationOK {
|
||
reasons = append(reasons, "没有针对该快照的有效授权")
|
||
}
|
||
if len(reasons) > 0 {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_resume_blocked", "reasons": reasons})
|
||
return
|
||
}
|
||
|
||
now := time.Now().UTC()
|
||
// 幂等键按源作业固定:重复点击返回同一个续作作业,而不是每次新建。
|
||
jobID := sourceID + "-resume"
|
||
var kind governance.LibraryJobKind
|
||
queueType := ""
|
||
switch source.Kind {
|
||
case governance.LibraryJobSync:
|
||
kind, queueType = governance.LibraryJobSync, librarySyncJobType
|
||
case governance.LibraryJobRestore:
|
||
kind, queueType = governance.LibraryJobRestore, libraryRestoreJobType
|
||
default:
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "封存作业由封存入口重发,不走续作"})
|
||
return
|
||
}
|
||
// F15:续作是**同一个意图的新执行代次**,不是新意图。沿用原作业的 operation,
|
||
// 维护窗口、预备份、写集合的谱系都还挂在同一条意图上;新作业行只是这次执行的载体,
|
||
// worker 起手会为它开一个更大的 epoch(旧代次从此不能写)。
|
||
operationID := source.OperationID
|
||
if operationID == "" {
|
||
// 迁移前的老作业没有 operation:按原作业身份补一条(同键同摘要 ⇒ 幂等,
|
||
// 不会因为点了两次续作而造出两条意图)。
|
||
intent, intentErr := ensureLibraryIntent(r.Context(), store, source.Scope, kind,
|
||
source.TargetSnapshotID, digest, identity.Username, "resume:"+sourceID, now)
|
||
if intentErr != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register resume intent"})
|
||
return
|
||
}
|
||
operationID = intent.Operation.ID
|
||
}
|
||
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
|
||
ID: jobID, Scope: source.Scope, Kind: kind,
|
||
IdempotencyKey: "library.job:" + operationID + ":resume",
|
||
RequestDigest: digest, OperationID: operationID,
|
||
TargetSnapshotID: source.TargetSnapshotID, Actor: identity.Username,
|
||
}, governance.SyncStages(), now)
|
||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "该目标已有在飞作业"})
|
||
return
|
||
}
|
||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"error": "同一续作请求参数不一致"})
|
||
return
|
||
}
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create resumed job"})
|
||
return
|
||
}
|
||
if reused {
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "resumed_from": sourceID, "reused": true})
|
||
return
|
||
}
|
||
// 写入已完成、只差登记版本的情况:续作只做幂等收尾,不重写生产。
|
||
body := map[string]any{"library_job_id": job.ID, "scope": source.Scope,
|
||
"target_snapshot_id": source.TargetSnapshotID}
|
||
if source.Stage == "finalize" && source.SnapshotID != "" {
|
||
body["finalize_only"] = true
|
||
}
|
||
payload, _ := json.Marshal(body)
|
||
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||
ID: governance.QueueJobIDFor(job.ID), Type: queueType, Payload: payload, AvailableAt: now,
|
||
MaxAttempts: 1, IdempotencyKey: "library.resume.queue:" + job.ID,
|
||
}, now)
|
||
if err != nil || queueJob == nil {
|
||
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败", now)
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue resumed job"})
|
||
return
|
||
}
|
||
// 附不上队列号就等于作业没人跑:必须显式失败并取消,不能静默返回 202。
|
||
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
|
||
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列归属写入失败", now)
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach resumed job"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "resumed_from": sourceID, "reused": false})
|
||
}
|