5e1482e349
build / build (push) Successful in 1m40s
根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关, 页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现, 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动: - 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口), cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize), 与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效, 同 target+scope+job 幂等)。 - worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写 library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功); 中断接管按同一作业 id 收掉维护窗口。 - API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整 整库目标、最近证据、失败原因),OpenAPI 同步。 - 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底, 证据存在时以证据为准。 - 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询),作业类型补 verify 文案。 测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定); 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
220 lines
8.8 KiB
Go
220 lines
8.8 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"net/http"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// 恢复验证:页面点一次"跑恢复验证",后端建一个持久作业,worker 用与 CLI 同一份实现
|
||
// (internal/libraryverify)真实执行:捕获目标 → 基线校验 → 写 canary 证明校验有效 →
|
||
// 按预备份恢复 → 恢复后复核。通过与否只由这条证据表回答,页面不接受手填开关。
|
||
const libraryVerifyJobType = "library.verify"
|
||
|
||
type libraryVerifyRequest struct {
|
||
Scope string `json:"scope"`
|
||
}
|
||
|
||
// libraryVerificationView 是页面需要的全部事实:当前目标是谁、最近一次验证结果、
|
||
// 证据是否对当前目标成立、以及同步闸口能不能据此放行。
|
||
type libraryVerificationView struct {
|
||
Scope string `json:"scope"`
|
||
Target libraryVerificationTarget `json:"target"`
|
||
Verified bool `json:"verified"`
|
||
Reason string `json:"reason"`
|
||
Latest *governance.LibraryRestoreVerification `json:"latest,omitempty"`
|
||
History []governance.LibraryRestoreVerification `json:"history"`
|
||
Running bool `json:"running"`
|
||
JobID string `json:"job_id,omitempty"`
|
||
}
|
||
|
||
type libraryVerificationTarget struct {
|
||
Instance string `json:"instance"`
|
||
Project string `json:"project"`
|
||
InternalKB string `json:"internal_kb"`
|
||
ExternalKB string `json:"external_kb"`
|
||
Index string `json:"index"`
|
||
S3Bucket string `json:"s3_bucket"`
|
||
TenantID uint64 `json:"tenant_id"`
|
||
DatabaseSet bool `json:"database_configured"`
|
||
}
|
||
|
||
// libraryVerifyEnqueue 投递一次恢复验证作业(管理员)。
|
||
//
|
||
// 它不签发发布授权、不改版本:验证只回答"当前目标的预备份能不能恢复"。
|
||
// 目标是部署配置,不接受请求体指定——否则一次点击就能验证别的库并把证据挂到当前目标上。
|
||
func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||
if r.Method != http.MethodPost {
|
||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||
return
|
||
}
|
||
identity, ok := sessionFrom(r.Context())
|
||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "恢复验证仅管理员可发起"})
|
||
return
|
||
}
|
||
var request libraryVerifyRequest
|
||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||
return
|
||
}
|
||
if request.Scope != "internal" && request.Scope != "external" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_verify_unavailable",
|
||
"error": "恢复验证存储不可用"})
|
||
return
|
||
}
|
||
// 验证要停写入者并在目标上写 canary:维护中或闸口关闭时直接拒绝,
|
||
// 不让页面投一个 worker 必然失败的作业。
|
||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||
writeJSON(w, http.StatusConflict, body)
|
||
return
|
||
}
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||
"code": "library_verify_blocked",
|
||
"reasons": []string{
|
||
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary",
|
||
},
|
||
})
|
||
return
|
||
}
|
||
digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobVerify,
|
||
map[string]any{"scope": request.Scope})
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"})
|
||
return
|
||
}
|
||
now := time.Now().UTC()
|
||
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
|
||
ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
|
||
Kind: governance.LibraryJobVerify,
|
||
IdempotencyKey: "library.verify:" + request.Scope + ":" + now.Format("20060102T150405.000000000"),
|
||
RequestDigest: digest, Actor: identity.Username,
|
||
}, governance.VerifyStages(), false, now)
|
||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active",
|
||
"error": "该目标已有在飞的写作业(同步/恢复/基线/验证互斥),等它结束再发起"})
|
||
return
|
||
}
|
||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
|
||
"error": "同一幂等键对应了不同的请求"})
|
||
return
|
||
}
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify job"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": reused})
|
||
}
|
||
|
||
// libraryVerifications 读当前目标的恢复验证证据(页面轮询用)。
|
||
func libraryVerifications(w http.ResponseWriter, r *http.Request, config Config) {
|
||
scope, err := scopeFromQuery(r)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||
return
|
||
}
|
||
view := libraryVerificationView{
|
||
Scope: scope,
|
||
History: []governance.LibraryRestoreVerification{},
|
||
Target: libraryVerificationTarget{
|
||
Instance: config.LibraryTargetInstance,
|
||
Project: config.LibraryTargetProject,
|
||
InternalKB: config.LibraryProductionInternalKB,
|
||
ExternalKB: config.LibraryProductionExternalKB,
|
||
Index: config.LibraryProductionIndex,
|
||
S3Bucket: config.LibraryProductionS3Bucket,
|
||
TenantID: config.LibraryProductionTenantID,
|
||
DatabaseSet: config.LibraryProductionDatabaseURL != "",
|
||
},
|
||
}
|
||
store, ok := config.Store.(interface {
|
||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||
LibraryRestoreVerifications(context.Context, string, int) ([]governance.LibraryRestoreVerification, error)
|
||
})
|
||
if !ok {
|
||
view.Reason = "当前部署没有恢复验证存储:无法读取证据"
|
||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||
return
|
||
}
|
||
latest, found, err := store.LatestLibraryRestoreVerification(r.Context(), scope)
|
||
if err != nil {
|
||
view.Reason = "读取恢复验证证据失败:" + err.Error()
|
||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||
return
|
||
}
|
||
if found {
|
||
view.Latest = &latest
|
||
view.JobID = latest.JobID
|
||
}
|
||
if history, err := store.LibraryRestoreVerifications(r.Context(), scope, 10); err == nil {
|
||
view.History = history
|
||
}
|
||
view.Verified, view.Reason = governance.LibraryVerificationSatisfied(latest, scope)
|
||
if !found {
|
||
view.Verified = false
|
||
view.Reason = "还没有对当前目标做过恢复验证:点「跑一次恢复验证」(真实备份 → canary → 恢复 → 复核)"
|
||
}
|
||
writeJSON(w, http.StatusOK, view)
|
||
}
|
||
|
||
// withRestoreEvidence 把"当前目标是否有真实恢复证据"注入 Config 的副本,
|
||
// 让同步/恢复闸口在同一个请求里按证据判断,而不是读一个可能被手填的 env。
|
||
// 读不到证据时不改写任何东西:闸口照旧因为缺证据而拒绝。
|
||
func (c Config) withRestoreEvidence(ctx context.Context, scope string) Config {
|
||
if ok, err := c.restoreEvidence(ctx, scope); err == nil {
|
||
c.restoreVerified = ok
|
||
c.restoreEvidenceReason = ""
|
||
} else {
|
||
c.restoreVerified = false
|
||
c.restoreEvidenceReason = err.Error()
|
||
}
|
||
return c
|
||
}
|
||
|
||
// restoreEvidence 读当前目标的最近一条证据并给出它能不能支撑"可恢复"。
|
||
func (c Config) restoreEvidence(ctx context.Context, scope string) (bool, error) {
|
||
store, ok := c.Store.(interface {
|
||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||
})
|
||
if !ok {
|
||
return false, errors.New("没有已核验的生产恢复演练证据")
|
||
}
|
||
latest, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||
if err != nil {
|
||
return false, fmt.Errorf("读取恢复验证证据失败:%w", err)
|
||
}
|
||
if !found {
|
||
return false, errors.New("当前目标还没有恢复验证证据:先在「生产保护与恢复」里跑一次恢复验证")
|
||
}
|
||
ok, reason := governance.LibraryVerificationSatisfied(latest, scope)
|
||
if !ok {
|
||
return false, errors.New(reason)
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
// restoreGateReasons 是闸口对"恢复可验证"的统一判断:证据优先,
|
||
// 历史 env 覆盖仅在证据缺失时兜底(页面不再写入它)。
|
||
func restoreGateReasons(config Config) []string {
|
||
if config.restoreVerified || config.ProductionBackupRestoreVerified {
|
||
return nil
|
||
}
|
||
reason := config.restoreEvidenceReason
|
||
if reason == "" {
|
||
reason = "没有已核验的生产恢复演练证据"
|
||
}
|
||
return []string{reason}
|
||
}
|