根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关, 页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现, 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动: - 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口), cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize), 与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效, 同 target+scope+job 幂等)。 - worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写 library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功); 中断接管按同一作业 id 收掉维护窗口。 - API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整 整库目标、最近证据、失败原因),OpenAPI 同步。 - 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底, 证据存在时以证据为准。 - 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询),作业类型补 verify 文案。 测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定); 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
This commit is contained in:
@@ -24,6 +24,9 @@ type BackupScheduleStore interface {
|
||||
type BackupHost interface {
|
||||
Available() bool
|
||||
Target() string
|
||||
// Endpoint 是挂载的 ssh 配置里该别名真实拨号的 host:port;页面据此证明
|
||||
// "连的是哪条通道",而不是只显示一个别名。
|
||||
Endpoint() string
|
||||
Show(context.Context) (backuphost.ShowResult, error)
|
||||
Apply(context.Context, bool, string, int, int) (backuphost.ApplyResult, error)
|
||||
RunNow(context.Context) (backuphost.RunResult, error)
|
||||
@@ -82,8 +85,11 @@ type backupScheduleView struct {
|
||||
// HostConfigured/HostReachable 区分"没接线"与"接线了但连不上"。
|
||||
HostConfigured bool `json:"host_configured"`
|
||||
HostTarget string `json:"host_target"`
|
||||
HostReachable bool `json:"host_reachable"`
|
||||
HostError string `json:"host_error"`
|
||||
// HostEndpoint 是这条别名真实拨号的 host:port(来自挂载的 ssh 配置)。
|
||||
// 连接失败时它就是"到底往哪连"的唯一证据,不能只给一个别名让人猜。
|
||||
HostEndpoint string `json:"host_endpoint"`
|
||||
HostReachable bool `json:"host_reachable"`
|
||||
HostError string `json:"host_error"`
|
||||
|
||||
HostScheduleEnabled bool `json:"host_schedule_enabled"`
|
||||
HostScheduleTime string `json:"host_schedule_time"`
|
||||
@@ -281,6 +287,7 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
|
||||
}
|
||||
view.HostConfigured = config.BackupHost.Available()
|
||||
view.HostTarget = config.BackupHost.Target()
|
||||
view.HostEndpoint = config.BackupHost.Endpoint()
|
||||
if !view.HostConfigured {
|
||||
view.InSync = false
|
||||
view.HostError = "目标主机凭据未挂载(BACKUP_HOST_SSH_CONFIG / BACKUP_HOST_TARGET)"
|
||||
@@ -293,6 +300,9 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
|
||||
if err != nil {
|
||||
view.HostReachable = false
|
||||
view.HostError = err.Error()
|
||||
if view.HostEndpoint != "" {
|
||||
view.HostError += "(拨号地址 " + view.HostEndpoint + ")"
|
||||
}
|
||||
view.InSync = false
|
||||
} else {
|
||||
view.HostReachable = true
|
||||
|
||||
@@ -46,6 +46,7 @@ func (store *fakeBackupScheduleStore) MarkBackupScheduleApplied(_ context.Contex
|
||||
type fakeBackupHost struct {
|
||||
available bool
|
||||
target string
|
||||
endpoint string
|
||||
show backuphost.ShowResult
|
||||
showErr error
|
||||
apply backuphost.ApplyResult
|
||||
@@ -59,6 +60,9 @@ type fakeBackupHost struct {
|
||||
|
||||
func (host *fakeBackupHost) Available() bool { return host.available }
|
||||
func (host *fakeBackupHost) Target() string { return host.target }
|
||||
func (host *fakeBackupHost) Endpoint() string {
|
||||
return host.endpoint
|
||||
}
|
||||
func (host *fakeBackupHost) Show(context.Context) (backuphost.ShowResult, error) {
|
||||
return host.show, host.showErr
|
||||
}
|
||||
@@ -166,6 +170,48 @@ func TestHandleBackupScheduleHostFlagsDrift(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBackupScheduleHostUnreachableSeparatesConnectionFromHostState(t *testing.T) {
|
||||
// 现场原错:ssh connect to host 120.76.217.51 port 22 Operation timed out。
|
||||
// 连接没建立 ⇒ 既不能报"脚本缺失",也不能报"计划未下发",只能报不可达 + 拨号地址。
|
||||
host := &fakeBackupHost{
|
||||
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
|
||||
showErr: errors.New("目标主机 weknora-test 不可达(ssh: connect to host 120.76.217.51 port 22: Operation timed out);未执行任何主机侧脚本,脚本与计划状态未知"),
|
||||
}
|
||||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||||
recorder := httptest.NewRecorder()
|
||||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||||
data := decodeData(t, recorder)
|
||||
if data["host_reachable"] != false || data["in_sync"] != false {
|
||||
t.Fatalf("连不上时不能报告可达或已同步: %+v", data)
|
||||
}
|
||||
message := data["host_error"].(string)
|
||||
if !strings.Contains(message, "不可达") || !strings.Contains(message, "100.85.111.121:22") {
|
||||
t.Fatalf("应报不可达并给出真实拨号地址: %q", message)
|
||||
}
|
||||
if strings.Contains(message, "脚本缺失") || strings.Contains(message, "未下发") {
|
||||
t.Fatalf("未观测到的脚本/计划状态不能当成结论: %q", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBackupScheduleHostReportsMissingScriptOnlyAfterConnecting(t *testing.T) {
|
||||
// 真正的脚本缺失:ssh 已连上,主机回的是脚本不存在 ⇒ 这时才允许说脚本缺失。
|
||||
host := &fakeBackupHost{
|
||||
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
|
||||
showErr: errors.New("已连上 weknora-test,但主机侧命令失败: bash: /opt/weknora-test/install_backup_schedule.sh: No such file or directory"),
|
||||
}
|
||||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||||
recorder := httptest.NewRecorder()
|
||||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||||
data := decodeData(t, recorder)
|
||||
message := data["host_error"].(string)
|
||||
if !strings.Contains(message, "已连上") || !strings.Contains(message, "No such file") {
|
||||
t.Fatalf("已连接但脚本缺失时应原样带回主机原因: %q", message)
|
||||
}
|
||||
if strings.Contains(message, "不可达") {
|
||||
t.Fatalf("已连上就不能说不可达: %q", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBackupScheduleHostUnconfiguredDegrades(t *testing.T) {
|
||||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, &fakeBackupHost{available: false})
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
@@ -642,9 +642,7 @@ func librarySyncGateWithGrant(config Config, production libraryKnownView, prebac
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
gate.Reasons = append(gate.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
gate.Reasons = append(gate.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
|
||||
}
|
||||
gate.Reasons = append(gate.Reasons, restoreGateReasons(config)...)
|
||||
// 生产端点以"整库目标是否配置"为准:整库同步直接写 PG/ES/对象,
|
||||
// 不依赖 WeKnora 的 HTTP 端,所以不能拿它是否配置当门禁。
|
||||
if strings.TrimSpace(config.LibraryProductionDatabaseURL) == "" {
|
||||
@@ -814,7 +812,8 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi
|
||||
|
||||
grant, hasGrant := activeGrantFor(r.Context(), config, scope, target)
|
||||
overview.Authorization = authorizationViewOf(grant, hasGrant)
|
||||
overview.Gate = librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, target, hasGrant)
|
||||
// 闸口按当前目标的真实恢复证据判断(没有证据时也接受历史 env 覆盖)。
|
||||
overview.Gate = librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), scope), overview.Production, overview.Prebackup, target, hasGrant)
|
||||
writeJSON(w, http.StatusOK, overview)
|
||||
}
|
||||
|
||||
@@ -963,7 +962,7 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
return
|
||||
}
|
||||
}
|
||||
gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant)
|
||||
gate := librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), request.Scope), overview.Production, overview.Prebackup, request.Target, hasGrant)
|
||||
if !gate.SyncAllowed {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||||
"code": "library_sync_blocked", "reasons": gate.Reasons,
|
||||
|
||||
@@ -71,9 +71,7 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
reasons = append(reasons, "生产写入闸口关闭")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||||
}
|
||||
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), request.Scope))...)
|
||||
authorizationOK := false
|
||||
if storeWithAuth, ok := config.Store.(interface {
|
||||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||||
@@ -169,9 +167,7 @@ func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config C
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
view.Reasons = append(view.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
|
||||
}
|
||||
view.Reasons = append(view.Reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), scope))...)
|
||||
digest, err := libraryRestoreParamsDigest(scope, snapshotID)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||||
@@ -228,9 +224,7 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
reasons = append(reasons, "生产写入闸口关闭")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||||
}
|
||||
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), source.Scope))...)
|
||||
var digest string
|
||||
switch source.Kind {
|
||||
case governance.LibraryJobSync:
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 恢复验证:页面点一次"跑恢复验证",后端建一个持久作业,worker 用与 CLI 同一份实现
|
||||
// (internal/libraryverify)真实执行:捕获目标 → 基线校验 → 写 canary 证明校验有效 →
|
||||
// 按预备份恢复 → 恢复后复核。通过与否只由这条证据表回答,页面不接受手填开关。
|
||||
const libraryVerifyJobType = "library.verify"
|
||||
|
||||
type libraryVerifyRequest struct {
|
||||
Scope string `json:"scope"`
|
||||
}
|
||||
|
||||
// libraryVerificationView 是页面需要的全部事实:当前目标是谁、最近一次验证结果、
|
||||
// 证据是否对当前目标成立、以及同步闸口能不能据此放行。
|
||||
type libraryVerificationView struct {
|
||||
Scope string `json:"scope"`
|
||||
Target libraryVerificationTarget `json:"target"`
|
||||
Verified bool `json:"verified"`
|
||||
Reason string `json:"reason"`
|
||||
Latest *governance.LibraryRestoreVerification `json:"latest,omitempty"`
|
||||
History []governance.LibraryRestoreVerification `json:"history"`
|
||||
Running bool `json:"running"`
|
||||
JobID string `json:"job_id,omitempty"`
|
||||
}
|
||||
|
||||
type libraryVerificationTarget struct {
|
||||
Instance string `json:"instance"`
|
||||
Project string `json:"project"`
|
||||
InternalKB string `json:"internal_kb"`
|
||||
ExternalKB string `json:"external_kb"`
|
||||
Index string `json:"index"`
|
||||
S3Bucket string `json:"s3_bucket"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
DatabaseSet bool `json:"database_configured"`
|
||||
}
|
||||
|
||||
// libraryVerifyEnqueue 投递一次恢复验证作业(管理员)。
|
||||
//
|
||||
// 它不签发发布授权、不改版本:验证只回答"当前目标的预备份能不能恢复"。
|
||||
// 目标是部署配置,不接受请求体指定——否则一次点击就能验证别的库并把证据挂到当前目标上。
|
||||
func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||||
return
|
||||
}
|
||||
identity, ok := sessionFrom(r.Context())
|
||||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "恢复验证仅管理员可发起"})
|
||||
return
|
||||
}
|
||||
var request libraryVerifyRequest
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return
|
||||
}
|
||||
if request.Scope != "internal" && request.Scope != "external" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||||
return
|
||||
}
|
||||
store, ok := libraryStore(config)
|
||||
if !ok {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_verify_unavailable",
|
||||
"error": "恢复验证存储不可用"})
|
||||
return
|
||||
}
|
||||
// 验证要停写入者并在目标上写 canary:维护中或闸口关闭时直接拒绝,
|
||||
// 不让页面投一个 worker 必然失败的作业。
|
||||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||||
writeJSON(w, http.StatusConflict, body)
|
||||
return
|
||||
}
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||||
"code": "library_verify_blocked",
|
||||
"reasons": []string{
|
||||
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary",
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobVerify,
|
||||
map[string]any{"scope": request.Scope})
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"})
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
|
||||
ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
|
||||
Kind: governance.LibraryJobVerify,
|
||||
IdempotencyKey: "library.verify:" + request.Scope + ":" + now.Format("20060102T150405.000000000"),
|
||||
RequestDigest: digest, Actor: identity.Username,
|
||||
}, governance.VerifyStages(), false, now)
|
||||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active",
|
||||
"error": "该目标已有在飞的写作业(同步/恢复/基线/验证互斥),等它结束再发起"})
|
||||
return
|
||||
}
|
||||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
|
||||
"error": "同一幂等键对应了不同的请求"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify job"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": reused})
|
||||
}
|
||||
|
||||
// libraryVerifications 读当前目标的恢复验证证据(页面轮询用)。
|
||||
func libraryVerifications(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
scope, err := scopeFromQuery(r)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
view := libraryVerificationView{
|
||||
Scope: scope,
|
||||
History: []governance.LibraryRestoreVerification{},
|
||||
Target: libraryVerificationTarget{
|
||||
Instance: config.LibraryTargetInstance,
|
||||
Project: config.LibraryTargetProject,
|
||||
InternalKB: config.LibraryProductionInternalKB,
|
||||
ExternalKB: config.LibraryProductionExternalKB,
|
||||
Index: config.LibraryProductionIndex,
|
||||
S3Bucket: config.LibraryProductionS3Bucket,
|
||||
TenantID: config.LibraryProductionTenantID,
|
||||
DatabaseSet: config.LibraryProductionDatabaseURL != "",
|
||||
},
|
||||
}
|
||||
store, ok := config.Store.(interface {
|
||||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
LibraryRestoreVerifications(context.Context, string, int) ([]governance.LibraryRestoreVerification, error)
|
||||
})
|
||||
if !ok {
|
||||
view.Reason = "当前部署没有恢复验证存储:无法读取证据"
|
||||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||||
return
|
||||
}
|
||||
latest, found, err := store.LatestLibraryRestoreVerification(r.Context(), scope)
|
||||
if err != nil {
|
||||
view.Reason = "读取恢复验证证据失败:" + err.Error()
|
||||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||||
return
|
||||
}
|
||||
if found {
|
||||
view.Latest = &latest
|
||||
view.JobID = latest.JobID
|
||||
}
|
||||
if history, err := store.LibraryRestoreVerifications(r.Context(), scope, 10); err == nil {
|
||||
view.History = history
|
||||
}
|
||||
view.Verified, view.Reason = governance.LibraryVerificationSatisfied(latest, scope)
|
||||
if !found {
|
||||
view.Verified = false
|
||||
view.Reason = "还没有对当前目标做过恢复验证:点「跑一次恢复验证」(真实备份 → canary → 恢复 → 复核)"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, view)
|
||||
}
|
||||
|
||||
// withRestoreEvidence 把"当前目标是否有真实恢复证据"注入 Config 的副本,
|
||||
// 让同步/恢复闸口在同一个请求里按证据判断,而不是读一个可能被手填的 env。
|
||||
// 读不到证据时不改写任何东西:闸口照旧因为缺证据而拒绝。
|
||||
func (c Config) withRestoreEvidence(ctx context.Context, scope string) Config {
|
||||
if ok, err := c.restoreEvidence(ctx, scope); err == nil {
|
||||
c.restoreVerified = ok
|
||||
c.restoreEvidenceReason = ""
|
||||
} else {
|
||||
c.restoreVerified = false
|
||||
c.restoreEvidenceReason = err.Error()
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// restoreEvidence 读当前目标的最近一条证据并给出它能不能支撑"可恢复"。
|
||||
func (c Config) restoreEvidence(ctx context.Context, scope string) (bool, error) {
|
||||
store, ok := c.Store.(interface {
|
||||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
})
|
||||
if !ok {
|
||||
return false, errors.New("没有已核验的生产恢复演练证据")
|
||||
}
|
||||
latest, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("读取恢复验证证据失败:%w", err)
|
||||
}
|
||||
if !found {
|
||||
return false, errors.New("当前目标还没有恢复验证证据:先在「生产保护与恢复」里跑一次恢复验证")
|
||||
}
|
||||
ok, reason := governance.LibraryVerificationSatisfied(latest, scope)
|
||||
if !ok {
|
||||
return false, errors.New(reason)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// restoreGateReasons 是闸口对"恢复可验证"的统一判断:证据优先,
|
||||
// 历史 env 覆盖仅在证据缺失时兜底(页面不再写入它)。
|
||||
func restoreGateReasons(config Config) []string {
|
||||
if config.restoreVerified || config.ProductionBackupRestoreVerified {
|
||||
return nil
|
||||
}
|
||||
reason := config.restoreEvidenceReason
|
||||
if reason == "" {
|
||||
reason = "没有已核验的生产恢复演练证据"
|
||||
}
|
||||
return []string{reason}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 恢复验证的页面契约:
|
||||
// - 只有管理员能发起(它会停写入者并在目标上写 canary);
|
||||
// - 闸口关闭 / 维护中时直接拒绝,不投一个 worker 必然失败的作业;
|
||||
// - 证据只由真实执行写入:没有证据 ⇒ verified=false,过期/换目标 ⇒ 失效。
|
||||
//
|
||||
// 这里不模拟"真跑一次恢复"(那需要真实目标);执行体本身在 internal/libraryverify,
|
||||
// 由库内 CLI 与 worker 共用,属于隔离 fixture 之外的集成范围。
|
||||
|
||||
type verificationStore struct {
|
||||
fakeLibraryStore
|
||||
verifications []governance.LibraryRestoreVerification
|
||||
}
|
||||
|
||||
func (s *verificationStore) LatestLibraryRestoreVerification(_ context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
for _, item := range s.verifications {
|
||||
if item.Scope == scope {
|
||||
return item, true, nil
|
||||
}
|
||||
}
|
||||
return governance.LibraryRestoreVerification{}, false, nil
|
||||
}
|
||||
|
||||
func (s *verificationStore) LibraryRestoreVerifications(_ context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
|
||||
items := make([]governance.LibraryRestoreVerification, 0, len(s.verifications))
|
||||
for _, item := range s.verifications {
|
||||
if item.Scope == scope {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
if limit > 0 && len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func verifyRequest(body, role, username string) *http.Request {
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/library/verifications", strings.NewReader(body))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
ctx := context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: username, Role: role})
|
||||
return request.WithContext(ctx)
|
||||
}
|
||||
|
||||
func TestLibraryVerifyRequiresAdmin(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
recorder := httptest.NewRecorder()
|
||||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleEngineer, "eng"),
|
||||
Config{Store: store, ProductionReleaseRequestsEnabled: true})
|
||||
if recorder.Code != http.StatusForbidden {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if store.createN != 0 {
|
||||
t.Fatal("non-admin must not create a verify job")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerifyRefusesWhenWriteGateClosed(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
recorder := httptest.NewRecorder()
|
||||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
|
||||
Config{Store: store, ProductionReleaseRequestsEnabled: false})
|
||||
if recorder.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
body := decodeBody(t, recorder)
|
||||
if body["code"] != "library_verify_blocked" {
|
||||
t.Fatalf("body=%v", body)
|
||||
}
|
||||
if store.createN != 0 {
|
||||
t.Fatal("closed gate must not create a verify job")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerifyEnqueuesVerifyJobWithCanaryStages(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
store.enqueueEchoRequestedID = true
|
||||
recorder := httptest.NewRecorder()
|
||||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
|
||||
Config{Store: store, ProductionReleaseRequestsEnabled: true})
|
||||
if recorder.Code != http.StatusAccepted {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if store.lastJob.Kind != governance.LibraryJobVerify || store.lastJob.Scope != "internal" {
|
||||
t.Fatalf("job=%#v", store.lastJob)
|
||||
}
|
||||
if !strings.HasPrefix(store.lastJob.ID, "libverify-internal-") {
|
||||
t.Fatalf("job id=%q", store.lastJob.ID)
|
||||
}
|
||||
// 阶段必须包含 canary:没有"写 canary 证明校验有效"这一步,验证就只是自证。
|
||||
want := governance.VerifyStages()
|
||||
if len(want) != 5 || want[2] != "canary" || want[3] != "restore" || want[4] != "finalize" {
|
||||
t.Fatalf("verify stages=%v", want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerificationsWithoutEvidenceIsNotVerified(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
body := decodeBody(t, recorder)
|
||||
if body["verified"] != false {
|
||||
t.Fatalf("verified must be false without evidence: %v", body)
|
||||
}
|
||||
if reason, _ := body["reason"].(string); !strings.Contains(reason, "还没有对当前目标做过恢复验证") {
|
||||
t.Fatalf("reason=%q", reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerificationsWithPassedEvidenceIsVerified(t *testing.T) {
|
||||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
ID: "libverify-internal-job1", Scope: "internal", Status: governance.LibraryVerificationPassed,
|
||||
SnapshotID: "snap-1", JobID: "job1", StartedAt: time.Now().UTC().Add(-time.Minute),
|
||||
FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
body := decodeBody(t, recorder)
|
||||
if body["verified"] != true {
|
||||
t.Fatalf("verified must be true with passed evidence: %v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerificationsFailedEvidenceIsNotVerified(t *testing.T) {
|
||||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
ID: "libverify-internal-job2", Scope: "internal", Status: governance.LibraryVerificationFailed,
|
||||
SnapshotID: "snap-2", JobID: "job2", Failure: "恢复后内容仍与预备份不一致",
|
||||
StartedAt: time.Now().UTC().Add(-time.Minute), FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
body := decodeBody(t, recorder)
|
||||
if body["verified"] != false {
|
||||
t.Fatalf("failed evidence must not verify: %v", body)
|
||||
}
|
||||
if reason, _ := body["reason"].(string); !strings.Contains(reason, "恢复后内容仍与预备份不一致") {
|
||||
t.Fatalf("reason=%q", reason)
|
||||
}
|
||||
}
|
||||
|
||||
// 同步闸口必须消费同一 target 的证据:有 passed 证据即放行,
|
||||
// 只有失败证据时仍然拒绝并给出失败原因(不是笼统"没有证据")。
|
||||
func TestLibrarySyncGateConsumesRestoreEvidence(t *testing.T) {
|
||||
base := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40}
|
||||
base.Store = &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1",
|
||||
JobID: "job1", FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
allowed := librarySyncGateWithGrant(base.withRestoreEvidence(context.Background(), "internal"),
|
||||
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
|
||||
if !allowed.SyncAllowed {
|
||||
t.Fatalf("gate should allow with passed evidence: %v", allowed.Reasons)
|
||||
}
|
||||
noEvidence := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
|
||||
Store: &verificationStore{}}.withRestoreEvidence(context.Background(), "internal")
|
||||
blocked := librarySyncGateWithGrant(noEvidence,
|
||||
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
|
||||
if blocked.SyncAllowed {
|
||||
t.Fatalf("gate without env and without evidence must block: %v", blocked.Reasons)
|
||||
}
|
||||
if len(blocked.Reasons) == 0 || !strings.Contains(strings.Join(blocked.Reasons, ";"), "恢复验证") {
|
||||
t.Fatalf("blocked reasons=%v", blocked.Reasons)
|
||||
}
|
||||
failedConfig := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
|
||||
Store: &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-2",
|
||||
JobID: "job2", Failure: "canary 后校验仍通过", FinishedAt: time.Now().UTC(),
|
||||
}}}}
|
||||
reason := strings.Join(restoreGateReasons(failedConfig.withRestoreEvidence(context.Background(), "internal")), ";")
|
||||
if !strings.Contains(reason, "canary 后校验仍通过") {
|
||||
t.Fatalf("失败证据的失败原因必须出现在闸口:%q", reason)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/pgdsn"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
@@ -33,6 +34,55 @@ type productionBindingView struct {
|
||||
ProductionBaseURL string `json:"production_base_url"`
|
||||
ProductionConnected bool `json:"production_connected"`
|
||||
ProductionMessage string `json:"production_message"`
|
||||
// LegacyScope 说明这条记录属于哪条发布路径。整库发布(封存/同步/恢复)不读它,
|
||||
// 只读下面的 WholeTarget;页面必须把两者分开写,否则会把旧文档级目标当成整库目标。
|
||||
LegacyScope string `json:"legacy_scope"`
|
||||
// WholeTarget 是整库发布真正写入的目标(来自部署环境 LIBRARY_PRODUCTION_*),
|
||||
// 与上面可编辑的文档级绑定是两条独立路径。
|
||||
WholeTarget libraryPublishTargetView `json:"whole_target"`
|
||||
}
|
||||
|
||||
// libraryPublishTargetView 是"整库发布到底写哪儿"的只读投影。
|
||||
// 为什么由后端给出:页面上的 kb 名称/文档数来自生产端列表接口,而整库写入用的是
|
||||
// 另一组环境配置;让页面自己拼会把两条路径混成一个。
|
||||
type libraryPublishTargetView struct {
|
||||
Configured bool `json:"configured"`
|
||||
Source string `json:"source"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
InternalKBID string `json:"internal_kb_id"`
|
||||
ExternalKBID string `json:"external_kb_id"`
|
||||
Database string `json:"database"`
|
||||
ElasticIndex string `json:"elastic_index"`
|
||||
ObjectBucket string `json:"object_bucket"`
|
||||
MissingReason string `json:"missing_reason"`
|
||||
}
|
||||
|
||||
// libraryPublishTarget 从 config 派生整库目标;缺项照实说缺哪一项,不猜默认值。
|
||||
func (config Config) libraryPublishTarget() libraryPublishTargetView {
|
||||
view := libraryPublishTargetView{
|
||||
Source: "部署环境 LIBRARY_PRODUCTION_*",
|
||||
TenantID: config.LibraryProductionTenantID,
|
||||
InternalKBID: config.LibraryProductionInternalKB,
|
||||
ExternalKBID: config.LibraryProductionExternalKB,
|
||||
Database: pgdsn.Endpoint(config.LibraryProductionDatabaseURL),
|
||||
ElasticIndex: config.LibraryProductionIndex,
|
||||
ObjectBucket: config.LibraryProductionS3Bucket,
|
||||
}
|
||||
missing := make([]string, 0, 4)
|
||||
if config.LibraryProductionDatabaseURL == "" {
|
||||
missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL")
|
||||
}
|
||||
if view.InternalKBID == "" {
|
||||
missing = append(missing, "LIBRARY_PRODUCTION_KB_INTERNAL")
|
||||
}
|
||||
if config.LibraryProductionIndex == "" {
|
||||
missing = append(missing, "LIBRARY_PRODUCTION_INDEX")
|
||||
}
|
||||
view.Configured = len(missing) == 0
|
||||
if len(missing) > 0 {
|
||||
view.MissingReason = "未配置:" + strings.Join(missing, "、")
|
||||
}
|
||||
return view
|
||||
}
|
||||
|
||||
// productionTargetBaseURL is the production endpoint the picker reads from: the
|
||||
@@ -79,6 +129,10 @@ func (config Config) productionBindingPayload(ctx context.Context) (productionBi
|
||||
InternalKBName: names[storedInternal], ExternalKBName: names[storedExternal],
|
||||
UpdatedBy: updatedBy, ProductionBaseURL: config.productionTargetBaseURL(ctx),
|
||||
ProductionConnected: message == "", ProductionMessage: message,
|
||||
// 这张表服务的是"文档级发布"(单篇文档写生产)。整库发布读的是部署环境
|
||||
// LIBRARY_PRODUCTION_*,两者不能互相顶替,所以页面必须分开显示。
|
||||
LegacyScope: "文档级发布(单篇文档写生产)· 旧路径",
|
||||
WholeTarget: config.libraryPublishTarget(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -178,3 +178,50 @@ func TestProductionBindingOptionsReportUnavailableReason(t *testing.T) {
|
||||
t.Fatalf("status = %d body %s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 页面必须说清"整库发布真正写哪个 KB/端点"。这条投影只能来自部署配置,
|
||||
// 不能从可编辑的文档级绑定推出来,也不能把口令带进响应。
|
||||
func TestLibraryPublishTargetIsDerivedFromDeploymentConfig(t *testing.T) {
|
||||
config := Config{
|
||||
// 只放形状,不放任何可用凭据:口令段是占位符,主机是 .invalid。
|
||||
LibraryProductionDatabaseURL: "postgres://governance@db.invalid:5432/weknora_test?sslmode=disable",
|
||||
LibraryProductionInternalKB: "a8e9c8ad-411c-4c53-8421-599d29e9b179",
|
||||
LibraryProductionExternalKB: "2e980686-2233-4c4e-b5a6-8b0069477b8e",
|
||||
LibraryProductionTenantID: 10000,
|
||||
LibraryProductionIndex: "weknora_test_chunks",
|
||||
LibraryProductionS3Bucket: "dobot-weknora-test",
|
||||
}
|
||||
target := config.libraryPublishTarget()
|
||||
if !target.Configured || target.MissingReason != "" {
|
||||
t.Fatalf("完整配置应判为已配置: %+v", target)
|
||||
}
|
||||
if target.InternalKBID != "a8e9c8ad-411c-4c53-8421-599d29e9b179" || target.TenantID != 10000 {
|
||||
t.Fatalf("必须回显实际整库目标而不是占位: %+v", target)
|
||||
}
|
||||
if target.Database != "db.invalid:5432/weknora_test" {
|
||||
t.Fatalf("端点应只含 host:port/db,得到 %q", target.Database)
|
||||
}
|
||||
// 回显只能是端点:连接串里的用户信息(口令、账号)一个字符都不能进响应。
|
||||
if strings.Contains(target.Database, "@") || strings.Contains(target.Source, "@") ||
|
||||
strings.Contains(target.Source, "postgres://") {
|
||||
t.Fatalf("响应里不能出现连接串用户信息: %+v", target)
|
||||
}
|
||||
if target.ElasticIndex != "weknora_test_chunks" || target.ObjectBucket != "dobot-weknora-test" {
|
||||
t.Fatalf("索引与桶也要显示: %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryPublishTargetNamesMissingConfigInsteadOfGuessing(t *testing.T) {
|
||||
target := Config{}.libraryPublishTarget()
|
||||
if target.Configured {
|
||||
t.Fatal("缺配置时不能报告已配置")
|
||||
}
|
||||
for _, key := range []string{"LIBRARY_PRODUCTION_DATABASE_URL", "LIBRARY_PRODUCTION_KB_INTERNAL"} {
|
||||
if !strings.Contains(target.MissingReason, key) {
|
||||
t.Fatalf("缺失说明应点名 %s,得到 %q", key, target.MissingReason)
|
||||
}
|
||||
}
|
||||
if target.InternalKBID != "" || target.Database != "(未配置)" {
|
||||
t.Fatalf("缺配置就不要编造目标: %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,6 +181,9 @@ type Config struct {
|
||||
LibraryProductionInternalKB string
|
||||
LibraryProductionExternalKB string
|
||||
LibraryProductionTenantID uint64
|
||||
// 整库发布还落在这些后端上;页面要显示"到底写哪儿",只给 KB ID 不够。
|
||||
LibraryProductionIndex string
|
||||
LibraryProductionS3Bucket string
|
||||
// LibraryScopeProbe 让测试注入"目标整库是否为空"的探测;留空即用真实连接。
|
||||
// 它不是运行时开关:生产部署不设置它,那次探测仍然直连目标库逐表计数。
|
||||
LibraryScopeProbe func(ctx context.Context, scope string) (bool, map[string]int64, error)
|
||||
@@ -198,8 +201,13 @@ type Config struct {
|
||||
RequireWeKnoraReady bool
|
||||
ProductionReleaseRequestsEnabled bool
|
||||
// TestAutoReviewPolicy 只用于展示:让运维在界面上看到"现在自动批准到什么程度"。
|
||||
TestAutoReviewPolicy string
|
||||
ProductionBackupRestoreVerified bool
|
||||
TestAutoReviewPolicy string
|
||||
ProductionBackupRestoreVerified bool
|
||||
// restoreVerified / restoreEvidenceReason 由请求期的证据读取填入(withRestoreEvidence):
|
||||
// 闸口按"当前目标是否有真实恢复验证证据"判断,而不是按可手填的 env。
|
||||
// 它们不可由配置注入,避免把证据变成另一个开关。
|
||||
restoreVerified bool
|
||||
restoreEvidenceReason string
|
||||
ProductionReleaseAuthorizerRef string
|
||||
ProductionReleaseAuthorizationToken string
|
||||
// DingtalkSources 枚举「我账号下能看到」的钉钉知识库与钉盘空间(只读,dws)。
|
||||
@@ -556,6 +564,14 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
mux.HandleFunc("POST /api/v1/library/baselines", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryBaselineEnqueue(w, r, config)
|
||||
})
|
||||
// 恢复验证:真实备份 → canary → 恢复 → 复核,通过后证据按当前目标登记,
|
||||
// 同步闸口据此放行(不再需要页面/环境变量手填 verified)。
|
||||
mux.HandleFunc("POST /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryVerifyEnqueue(w, r, config)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryVerifications(w, r, config)
|
||||
})
|
||||
// 全部 main 提交时间线(读本库缓存,不直接打 Gitea)
|
||||
mux.HandleFunc("GET /api/v1/version-history", func(w http.ResponseWriter, r *http.Request) {
|
||||
versionHistoryList(w, r, config)
|
||||
|
||||
Reference in New Issue
Block a user