姚章浩
b4add6c9b0
B1-R1: 演练写集改为受限 restore-drill（V2 信封/contract 1/schema 摘要，零对象、零索引、零本地映射，来源身份必填）。B2-R1: verify 作业先建不可变 operation 并携带 operation_id，同 job recover 的网关 claim 同时携带原 operation 与 recovery 身份。B5-R1: 前端按 scope 世代与动作序号丢弃迟到响应，不污染新 scope 或父级旧回调。harness 的 recovery 改由生产 jobs.Worker.RunOnce 消费并读回 jobs 终态；原失败窗口作业保持 failed，不改写历史。
297 lines
12 KiB
Go
297 lines
12 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"net/http"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// 恢复验证：页面点一次"跑恢复验证"，后端建一个持久作业，worker 用与 CLI 同一份实现
|
||
// （internal/libraryverify）真实执行：捕获目标 → 基线校验 → 写 canary 证明校验有效 →
|
||
// 按预备份恢复 → 恢复后复核。通过与否只由这条证据表回答，页面不接受手填开关。
|
||
const libraryVerifyJobType = "library.verify"
|
||
|
||
type libraryVerifyRequest struct {
|
||
Scope string `json:"scope"`
|
||
}
|
||
|
||
// libraryVerificationView 是页面需要的全部事实：当前目标是谁、最近一次验证结果、
|
||
// 证据是否对当前目标成立、以及同步闸口能不能据此放行。
|
||
type libraryVerificationView struct {
|
||
Scope string `json:"scope"`
|
||
Target libraryVerificationTarget `json:"target"`
|
||
Verified bool `json:"verified"`
|
||
Reason string `json:"reason"`
|
||
Latest *governance.LibraryRestoreVerification `json:"latest,omitempty"`
|
||
History []governance.LibraryRestoreVerification `json:"history"`
|
||
Running bool `json:"running"`
|
||
JobID string `json:"job_id,omitempty"`
|
||
State string `json:"state,omitempty"`
|
||
Stage string `json:"stage,omitempty"`
|
||
}
|
||
|
||
type libraryVerificationTarget struct {
|
||
Instance string `json:"instance"`
|
||
Project string `json:"project"`
|
||
InternalKB string `json:"internal_kb"`
|
||
ExternalKB string `json:"external_kb"`
|
||
Index string `json:"index"`
|
||
S3Bucket string `json:"s3_bucket"`
|
||
TenantID uint64 `json:"tenant_id"`
|
||
DatabaseSet bool `json:"database_configured"`
|
||
}
|
||
|
||
// libraryVerifyEnqueue 投递一次恢复验证作业（管理员）。
|
||
//
|
||
// 它不签发发布授权、不改版本：验证只回答"当前目标的预备份能不能恢复"。
|
||
// 目标是部署配置，不接受请求体指定——否则一次点击就能验证别的库并把证据挂到当前目标上。
|
||
func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||
if r.Method != http.MethodPost {
|
||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||
return
|
||
}
|
||
identity, ok := sessionFrom(r.Context())
|
||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "恢复验证仅管理员可发起"})
|
||
return
|
||
}
|
||
var request libraryVerifyRequest
|
||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||
return
|
||
}
|
||
if request.Scope != "internal" && request.Scope != "external" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_verify_unavailable",
|
||
"error": "恢复验证存储不可用"})
|
||
return
|
||
}
|
||
// 验证要停写入者并在目标上写 canary：维护中或闸口关闭时直接拒绝，
|
||
// 不让页面投一个 worker 必然失败的作业。
|
||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||
writeJSON(w, http.StatusConflict, body)
|
||
return
|
||
}
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||
"code": "library_verify_blocked",
|
||
"reasons": []string{
|
||
"生产写入闸口关闭（PRODUCTION_RELEASE_REQUESTS_ENABLED=false）：恢复验证要停写入者并在目标上写 canary",
|
||
},
|
||
})
|
||
return
|
||
}
|
||
digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobVerify,
|
||
map[string]any{"scope": request.Scope})
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"})
|
||
return
|
||
}
|
||
now := time.Now().UTC()
|
||
// 验证作业必须有和同步/恢复同一套不可变 operation：worker 的 owned attempt 从
|
||
// job.operation_id 取身份，恢复派发闸口和对象网关都按它核验。没有 operation 的
|
||
// verify 作业在真实恢复时会被 fail closed（审核 B2-R1）。
|
||
intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobVerify,
|
||
"", digest, identity.Username, "", now)
|
||
if intentErr != nil {
|
||
if isIntentConflict(intentErr) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
|
||
"error": "同一幂等键对应了不同的请求"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify operation"})
|
||
return
|
||
}
|
||
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
|
||
ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
|
||
Kind: governance.LibraryJobVerify,
|
||
IdempotencyKey: intent.JobKey,
|
||
RequestDigest: digest, OperationID: intent.Operation.ID, Actor: identity.Username,
|
||
}, governance.VerifyStages(), false, now)
|
||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active",
|
||
"error": "该目标已有在飞的写作业（同步/恢复/基线/验证互斥），等它结束再发起"})
|
||
return
|
||
}
|
||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
|
||
"error": "同一幂等键对应了不同的请求"})
|
||
return
|
||
}
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify job"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": reused})
|
||
}
|
||
|
||
// libraryVerifications 读当前目标的恢复验证证据（页面轮询用）。
|
||
func libraryVerifications(w http.ResponseWriter, r *http.Request, config Config) {
|
||
scope, err := scopeFromQuery(r)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||
return
|
||
}
|
||
view := libraryVerificationView{
|
||
Scope: scope,
|
||
History: []governance.LibraryRestoreVerification{},
|
||
Target: libraryVerificationTarget{
|
||
Instance: config.LibraryTargetInstance,
|
||
Project: config.LibraryTargetProject,
|
||
InternalKB: config.LibraryProductionInternalKB,
|
||
ExternalKB: config.LibraryProductionExternalKB,
|
||
Index: config.LibraryProductionIndex,
|
||
S3Bucket: config.LibraryProductionS3Bucket,
|
||
TenantID: config.LibraryProductionTenantID,
|
||
DatabaseSet: config.LibraryProductionDatabaseURL != "",
|
||
},
|
||
}
|
||
store, ok := config.Store.(interface {
|
||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||
LibraryRestoreVerifications(context.Context, string, int) ([]governance.LibraryRestoreVerification, error)
|
||
})
|
||
if !ok {
|
||
view.Reason = "当前部署没有恢复验证存储：无法读取证据"
|
||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||
return
|
||
}
|
||
latest, found, err := store.LatestLibraryRestoreVerification(r.Context(), scope)
|
||
if err != nil {
|
||
view.Reason = "读取恢复验证证据失败：" + err.Error()
|
||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||
return
|
||
}
|
||
if found {
|
||
view.Latest = &latest
|
||
view.JobID = latest.JobID
|
||
}
|
||
if history, err := store.LibraryRestoreVerifications(r.Context(), scope, 10); err == nil {
|
||
view.History = history
|
||
}
|
||
view.Verified, view.Reason = governance.LibraryVerificationSatisfied(latest, scope)
|
||
if !found {
|
||
view.Verified = false
|
||
view.Reason = "还没有对当前目标做过恢复验证：点「跑一次恢复验证」（真实备份 → canary → 恢复 → 复核）"
|
||
}
|
||
// 在飞作业必须从真实业务行读出来：旧实现只构造 running 字段却从不赋值，
|
||
// 页面因此在验收里看不到"已接受/正在执行"，按钮也不会进入执行态（审核 B4）。
|
||
if jobs, ok := config.Store.(interface {
|
||
ListLibraryJobs(context.Context, string, int) ([]governance.LibraryJob, error)
|
||
}); ok {
|
||
if items, err := jobs.ListLibraryJobs(r.Context(), scope, 20); err == nil {
|
||
var active *governance.LibraryJob
|
||
for index := range items {
|
||
item := items[index]
|
||
if item.Kind != governance.LibraryJobVerify || string(item.State) == "" {
|
||
continue
|
||
}
|
||
if !libraryJobActiveState(item.State) {
|
||
continue
|
||
}
|
||
if active == nil || item.CreatedAt.After(active.CreatedAt) {
|
||
active = &item
|
||
}
|
||
}
|
||
if active != nil {
|
||
view.Running = true
|
||
view.JobID = active.ID
|
||
view.State = string(active.State)
|
||
view.Stage = active.Stage
|
||
view.Reason = "恢复验证作业 " + active.ID + " 正在执行（" + active.Stage + "）；终态会在这里显示证据，页面会自动刷新"
|
||
}
|
||
}
|
||
}
|
||
writeJSON(w, http.StatusOK, view)
|
||
}
|
||
|
||
// libraryJobActiveState 只认仍在执行的业务状态：终态不是"在飞"。
|
||
func libraryJobActiveState(state governance.LibraryJobState) bool {
|
||
switch state {
|
||
case governance.LibraryJobPending, governance.LibraryJobRunning:
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// withRestoreEvidence 把"当前目标是否有真实恢复证据"注入 Config 的副本，
|
||
// 让同步/恢复闸口在同一个请求里按证据判断，而不是读一个可能被手填的 env。
|
||
//
|
||
// 证据优先级是不可交换的（审核 B3）：明确 failed/invalid 或读取失败都必须拒绝，
|
||
// 即使 env=true；历史 env 覆盖只在**明确没有证据行**（not-found）时兜底。
|
||
// 把读取失败混进 not-found 等于把"不知道"说成"满足"。
|
||
func (c Config) withRestoreEvidence(ctx context.Context, scope string) Config {
|
||
satisfied, missing, err := c.restoreEvidence(ctx, scope)
|
||
c.restoreEvidenceChecked = true
|
||
switch {
|
||
case err != nil:
|
||
c.restoreVerified = false
|
||
c.restoreEvidenceReason = err.Error()
|
||
case satisfied:
|
||
c.restoreVerified = true
|
||
c.restoreEvidenceReason = ""
|
||
case missing && c.ProductionBackupRestoreVerified:
|
||
c.restoreVerified = true
|
||
c.restoreEvidenceReason = ""
|
||
default:
|
||
c.restoreVerified = false
|
||
if missing {
|
||
c.restoreEvidenceReason = "当前目标还没有恢复演练证据：先在「生产保护与恢复」里跑一次恢复验证"
|
||
} else {
|
||
c.restoreEvidenceReason = "没有已核验的生产恢复演练证据"
|
||
}
|
||
}
|
||
return c
|
||
}
|
||
|
||
// restoreEvidence 读当前目标的最近一条证据并给出它能不能支撑"可恢复"。
|
||
// missing 只在"查询成功且确实没有这一行"时为 true；读取错误单独返回。
|
||
func (c Config) restoreEvidence(ctx context.Context, scope string) (satisfied bool, missing bool, err error) {
|
||
store, ok := c.Store.(interface {
|
||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||
})
|
||
if !ok {
|
||
// 存储没有证据能力（迁移前的部署形状）等同于"没有证据行"：
|
||
// 历史 env 覆盖仍可兜底，但一旦读得出证据行，失败/无效/读取错误都优先拒绝。
|
||
return false, true, nil
|
||
}
|
||
latest, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||
if err != nil {
|
||
return false, false, fmt.Errorf("读取恢复验证证据失败：%w", err)
|
||
}
|
||
if !found {
|
||
return false, true, nil
|
||
}
|
||
ok, reason := governance.LibraryVerificationSatisfied(latest, scope)
|
||
if !ok {
|
||
return false, false, errors.New(reason)
|
||
}
|
||
return true, false, nil
|
||
}
|
||
|
||
// restoreGateReasons 是闸口对"恢复可验证"的统一判断：只看 withRestoreEvidence 的结论。
|
||
// 这里不再 OR env：env 兜底只在证据缺失时、且已经在 withRestoreEvidence 里判定过。
|
||
func restoreGateReasons(config Config) []string {
|
||
if config.restoreVerified {
|
||
return nil
|
||
}
|
||
// 没有走过证据读取的调用点（历史路径/单元夹具）保持原有 env 语义；
|
||
// 一旦读过证据，上面的 withRestoreEvidence 已经把优先级判定完毕。
|
||
if !config.restoreEvidenceChecked && config.ProductionBackupRestoreVerified {
|
||
return nil
|
||
}
|
||
reason := config.restoreEvidenceReason
|
||
if reason == "" {
|
||
reason = "没有已核验的生产恢复演练证据"
|
||
}
|
||
return []string{reason}
|
||
}
|