5e1482e349
build / build (push) Successful in 1m40s
根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关, 页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现, 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动: - 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口), cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize), 与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效, 同 target+scope+job 幂等)。 - worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写 library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功); 中断接管按同一作业 id 收掉维护窗口。 - API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整 整库目标、最近证据、失败原因),OpenAPI 同步。 - 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底, 证据存在时以证据为准。 - 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询),作业类型补 verify 文案。 测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定); 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
194 lines
8.4 KiB
Go
194 lines
8.4 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// 恢复验证的页面契约:
|
||
// - 只有管理员能发起(它会停写入者并在目标上写 canary);
|
||
// - 闸口关闭 / 维护中时直接拒绝,不投一个 worker 必然失败的作业;
|
||
// - 证据只由真实执行写入:没有证据 ⇒ verified=false,过期/换目标 ⇒ 失效。
|
||
//
|
||
// 这里不模拟"真跑一次恢复"(那需要真实目标);执行体本身在 internal/libraryverify,
|
||
// 由库内 CLI 与 worker 共用,属于隔离 fixture 之外的集成范围。
|
||
|
||
type verificationStore struct {
|
||
fakeLibraryStore
|
||
verifications []governance.LibraryRestoreVerification
|
||
}
|
||
|
||
func (s *verificationStore) LatestLibraryRestoreVerification(_ context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
|
||
for _, item := range s.verifications {
|
||
if item.Scope == scope {
|
||
return item, true, nil
|
||
}
|
||
}
|
||
return governance.LibraryRestoreVerification{}, false, nil
|
||
}
|
||
|
||
func (s *verificationStore) LibraryRestoreVerifications(_ context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
|
||
items := make([]governance.LibraryRestoreVerification, 0, len(s.verifications))
|
||
for _, item := range s.verifications {
|
||
if item.Scope == scope {
|
||
items = append(items, item)
|
||
}
|
||
}
|
||
if limit > 0 && len(items) > limit {
|
||
items = items[:limit]
|
||
}
|
||
return items, nil
|
||
}
|
||
|
||
func verifyRequest(body, role, username string) *http.Request {
|
||
request := httptest.NewRequest(http.MethodPost, "/api/v1/library/verifications", strings.NewReader(body))
|
||
request.Header.Set("Content-Type", "application/json")
|
||
ctx := context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: username, Role: role})
|
||
return request.WithContext(ctx)
|
||
}
|
||
|
||
func TestLibraryVerifyRequiresAdmin(t *testing.T) {
|
||
store := &verificationStore{}
|
||
recorder := httptest.NewRecorder()
|
||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleEngineer, "eng"),
|
||
Config{Store: store, ProductionReleaseRequestsEnabled: true})
|
||
if recorder.Code != http.StatusForbidden {
|
||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||
}
|
||
if store.createN != 0 {
|
||
t.Fatal("non-admin must not create a verify job")
|
||
}
|
||
}
|
||
|
||
func TestLibraryVerifyRefusesWhenWriteGateClosed(t *testing.T) {
|
||
store := &verificationStore{}
|
||
recorder := httptest.NewRecorder()
|
||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
|
||
Config{Store: store, ProductionReleaseRequestsEnabled: false})
|
||
if recorder.Code != http.StatusServiceUnavailable {
|
||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||
}
|
||
body := decodeBody(t, recorder)
|
||
if body["code"] != "library_verify_blocked" {
|
||
t.Fatalf("body=%v", body)
|
||
}
|
||
if store.createN != 0 {
|
||
t.Fatal("closed gate must not create a verify job")
|
||
}
|
||
}
|
||
|
||
func TestLibraryVerifyEnqueuesVerifyJobWithCanaryStages(t *testing.T) {
|
||
store := &verificationStore{}
|
||
store.enqueueEchoRequestedID = true
|
||
recorder := httptest.NewRecorder()
|
||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
|
||
Config{Store: store, ProductionReleaseRequestsEnabled: true})
|
||
if recorder.Code != http.StatusAccepted {
|
||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||
}
|
||
if store.lastJob.Kind != governance.LibraryJobVerify || store.lastJob.Scope != "internal" {
|
||
t.Fatalf("job=%#v", store.lastJob)
|
||
}
|
||
if !strings.HasPrefix(store.lastJob.ID, "libverify-internal-") {
|
||
t.Fatalf("job id=%q", store.lastJob.ID)
|
||
}
|
||
// 阶段必须包含 canary:没有"写 canary 证明校验有效"这一步,验证就只是自证。
|
||
want := governance.VerifyStages()
|
||
if len(want) != 5 || want[2] != "canary" || want[3] != "restore" || want[4] != "finalize" {
|
||
t.Fatalf("verify stages=%v", want)
|
||
}
|
||
}
|
||
|
||
func TestLibraryVerificationsWithoutEvidenceIsNotVerified(t *testing.T) {
|
||
store := &verificationStore{}
|
||
recorder := httptest.NewRecorder()
|
||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||
libraryVerifications(recorder, request, Config{Store: store})
|
||
if recorder.Code != http.StatusOK {
|
||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||
}
|
||
body := decodeBody(t, recorder)
|
||
if body["verified"] != false {
|
||
t.Fatalf("verified must be false without evidence: %v", body)
|
||
}
|
||
if reason, _ := body["reason"].(string); !strings.Contains(reason, "还没有对当前目标做过恢复验证") {
|
||
t.Fatalf("reason=%q", reason)
|
||
}
|
||
}
|
||
|
||
func TestLibraryVerificationsWithPassedEvidenceIsVerified(t *testing.T) {
|
||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||
ID: "libverify-internal-job1", Scope: "internal", Status: governance.LibraryVerificationPassed,
|
||
SnapshotID: "snap-1", JobID: "job1", StartedAt: time.Now().UTC().Add(-time.Minute),
|
||
FinishedAt: time.Now().UTC(),
|
||
}}}
|
||
recorder := httptest.NewRecorder()
|
||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||
libraryVerifications(recorder, request, Config{Store: store})
|
||
body := decodeBody(t, recorder)
|
||
if body["verified"] != true {
|
||
t.Fatalf("verified must be true with passed evidence: %v", body)
|
||
}
|
||
}
|
||
|
||
func TestLibraryVerificationsFailedEvidenceIsNotVerified(t *testing.T) {
|
||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||
ID: "libverify-internal-job2", Scope: "internal", Status: governance.LibraryVerificationFailed,
|
||
SnapshotID: "snap-2", JobID: "job2", Failure: "恢复后内容仍与预备份不一致",
|
||
StartedAt: time.Now().UTC().Add(-time.Minute), FinishedAt: time.Now().UTC(),
|
||
}}}
|
||
recorder := httptest.NewRecorder()
|
||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||
libraryVerifications(recorder, request, Config{Store: store})
|
||
body := decodeBody(t, recorder)
|
||
if body["verified"] != false {
|
||
t.Fatalf("failed evidence must not verify: %v", body)
|
||
}
|
||
if reason, _ := body["reason"].(string); !strings.Contains(reason, "恢复后内容仍与预备份不一致") {
|
||
t.Fatalf("reason=%q", reason)
|
||
}
|
||
}
|
||
|
||
// 同步闸口必须消费同一 target 的证据:有 passed 证据即放行,
|
||
// 只有失败证据时仍然拒绝并给出失败原因(不是笼统"没有证据")。
|
||
func TestLibrarySyncGateConsumesRestoreEvidence(t *testing.T) {
|
||
base := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40}
|
||
base.Store = &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1",
|
||
JobID: "job1", FinishedAt: time.Now().UTC(),
|
||
}}}
|
||
allowed := librarySyncGateWithGrant(base.withRestoreEvidence(context.Background(), "internal"),
|
||
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
|
||
if !allowed.SyncAllowed {
|
||
t.Fatalf("gate should allow with passed evidence: %v", allowed.Reasons)
|
||
}
|
||
noEvidence := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
|
||
Store: &verificationStore{}}.withRestoreEvidence(context.Background(), "internal")
|
||
blocked := librarySyncGateWithGrant(noEvidence,
|
||
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
|
||
if blocked.SyncAllowed {
|
||
t.Fatalf("gate without env and without evidence must block: %v", blocked.Reasons)
|
||
}
|
||
if len(blocked.Reasons) == 0 || !strings.Contains(strings.Join(blocked.Reasons, ";"), "恢复验证") {
|
||
t.Fatalf("blocked reasons=%v", blocked.Reasons)
|
||
}
|
||
failedConfig := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
|
||
Store: &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-2",
|
||
JobID: "job2", Failure: "canary 后校验仍通过", FinishedAt: time.Now().UTC(),
|
||
}}}}
|
||
reason := strings.Join(restoreGateReasons(failedConfig.withRestoreEvidence(context.Background(), "internal")), ";")
|
||
if !strings.Contains(reason, "canary 后校验仍通过") {
|
||
t.Fatalf("失败证据的失败原因必须出现在闸口:%q", reason)
|
||
}
|
||
}
|