姚章浩
5e1482e349
build / build (push) Successful in 1m40s
根因：生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关， 页面没有任何入口去真跑一次恢复；CLI（librarydrill）又不是 worker 能复用的实现， 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动： - 共享实现 internal/libraryverify（维护隔离→捕获→基线校验→canary→恢复→复核→关窗口）， cmd/librarydrill 改为薄封装，CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业：kind + VerifyStages(capture/verify/canary/restore/finalize)， 与 sync/restore/baseline 共用同一把写互斥；迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications（按 target_id 绑定，换库/索引/桶旧证据自动失效， 同 target+scope+job 幂等）。 - worker：admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现，阶段实时写 library_job_stages，结束后写真实终态；通过写 passed 证据，失败写 failed+原因（不伪成功）； 中断接管按同一作业 id 收掉维护窗口。 - API：POST/GET /api/v1/library/verifications（管理员；闸口关闭/维护中拒绝；返回完整 整库目标、最近证据、失败原因），OpenAPI 同步。 - 闸口：同步/恢复/续作的证据判断改为读当前目标证据；历史 env 仅在证据缺失时兜底， 证据存在时以证据为准。 - 页面：生产保护与恢复面板加「恢复验证」块（完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询），作业类型补 verify 文案。 测试：make verify 16/16 全绿（go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定）； 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
402 lines
16 KiB
Go
402 lines
16 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/backuphost"
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
type fakeBackupScheduleStore struct {
|
||
schedule governance.BackupSchedule
|
||
loadErr error
|
||
saveErr error
|
||
saved []governance.BackupSchedule
|
||
appliedAt []string
|
||
}
|
||
|
||
func (store *fakeBackupScheduleStore) LoadBackupSchedule(context.Context) (governance.BackupSchedule, error) {
|
||
if store.loadErr != nil {
|
||
return governance.BackupSchedule{}, store.loadErr
|
||
}
|
||
return store.schedule, nil
|
||
}
|
||
|
||
func (store *fakeBackupScheduleStore) SaveBackupSchedule(_ context.Context, schedule governance.BackupSchedule, _ string, _ time.Time) error {
|
||
if store.saveErr != nil {
|
||
return store.saveErr
|
||
}
|
||
store.schedule = schedule
|
||
store.saved = append(store.saved, schedule)
|
||
return nil
|
||
}
|
||
|
||
func (store *fakeBackupScheduleStore) MarkBackupScheduleApplied(_ context.Context, summary string, _ time.Time) error {
|
||
store.appliedAt = append(store.appliedAt, summary)
|
||
return nil
|
||
}
|
||
|
||
type fakeBackupHost struct {
|
||
available bool
|
||
target string
|
||
endpoint string
|
||
show backuphost.ShowResult
|
||
showErr error
|
||
apply backuphost.ApplyResult
|
||
applyErr error
|
||
lastRun backuphost.RunResult
|
||
lastRunErr error
|
||
runErr error
|
||
applied int
|
||
runs int
|
||
}
|
||
|
||
func (host *fakeBackupHost) Available() bool { return host.available }
|
||
func (host *fakeBackupHost) Target() string { return host.target }
|
||
func (host *fakeBackupHost) Endpoint() string {
|
||
return host.endpoint
|
||
}
|
||
func (host *fakeBackupHost) Show(context.Context) (backuphost.ShowResult, error) {
|
||
return host.show, host.showErr
|
||
}
|
||
func (host *fakeBackupHost) Apply(context.Context, bool, string, int, int) (backuphost.ApplyResult, error) {
|
||
host.applied++
|
||
return host.apply, host.applyErr
|
||
}
|
||
func (host *fakeBackupHost) RunNow(context.Context) (backuphost.RunResult, error) {
|
||
host.runs++
|
||
return host.lastRun, host.runErr
|
||
}
|
||
func (host *fakeBackupHost) LastRun(context.Context) (backuphost.RunResult, error) {
|
||
return host.lastRun, host.lastRunErr
|
||
}
|
||
|
||
func backupConfig(store BackupScheduleStore, host BackupHost) Config {
|
||
return Config{
|
||
Store: nil,
|
||
BackupSchedule: store,
|
||
BackupHost: host,
|
||
BackupRuns: &BackupRunTracker{},
|
||
Now: func() time.Time { return time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) },
|
||
}
|
||
}
|
||
|
||
func decodeData(t *testing.T, recorder *httptest.ResponseRecorder) map[string]any {
|
||
t.Helper()
|
||
var payload struct {
|
||
Data map[string]any `json:"data"`
|
||
}
|
||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||
t.Fatalf("响应不是 JSON: %v（%s）", err, recorder.Body.String())
|
||
}
|
||
return payload.Data
|
||
}
|
||
|
||
var goldenSchedule = governance.BackupSchedule{
|
||
Enabled: true, ScheduleTime: "03:30", RetentionDays: 7, RetentionMonths: 3, UpdatedBy: "admin",
|
||
}
|
||
|
||
func TestHandleBackupScheduleReadsPlanWithoutProbingHost(t *testing.T) {
|
||
host := &fakeBackupHost{available: true, target: "weknora-test", showErr: errors.New("ssh: connect timed out")}
|
||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||
recorder := httptest.NewRecorder()
|
||
config.handleBackupSchedule(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule", nil))
|
||
if recorder.Code != http.StatusOK {
|
||
t.Fatalf("期望 200，得到 %d", recorder.Code)
|
||
}
|
||
data := decodeData(t, recorder)
|
||
if data["schedule_time"] != "03:30" || data["retention_days"] != float64(7) || data["enabled"] != true {
|
||
t.Fatalf("计划字段不对: %+v", data)
|
||
}
|
||
// 只读接口不探测主机：即使主机不可达也不应带错误（首屏不被 SSH 拖慢）。
|
||
if data["host_error"] != "" {
|
||
t.Fatalf("只读接口不应探测主机: %+v", data["host_error"])
|
||
}
|
||
if data["host_configured"] != true || data["host_target"] != "weknora-test" {
|
||
t.Fatalf("应报告已接线与目标主机: %+v", data)
|
||
}
|
||
}
|
||
|
||
func TestHandleBackupScheduleHostReportsPlanAndLastRun(t *testing.T) {
|
||
host := &fakeBackupHost{
|
||
available: true, target: "weknora-test",
|
||
show: backuphost.ShowResult{
|
||
ScheduleEnabled: true, ScheduleTime: "03:30", KeepDays: 7, KeepMonths: 3,
|
||
CronLines: []string{"30 03 * * * root /bin/bash /opt/weknora-test/logical-backup.sh"}, CronService: "active",
|
||
ScriptPresent: true,
|
||
},
|
||
lastRun: backuphost.RunResult{
|
||
Stamp: "20261009T052909Z", DurationSeconds: 33, TotalBytes: 778502144,
|
||
Databases: []string{"weknora_test"}, FailedDatabases: []string{},
|
||
},
|
||
}
|
||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||
recorder := httptest.NewRecorder()
|
||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||
data := decodeData(t, recorder)
|
||
if data["host_reachable"] != true || data["host_cron_service"] != "active" {
|
||
t.Fatalf("主机状态不对: %+v", data)
|
||
}
|
||
if data["last_run_stamp"] != "20261009T052909Z" || data["last_run_duration_seconds"] != float64(33) {
|
||
t.Fatalf("最近一次结果不对: %+v", data)
|
||
}
|
||
// 主机上的计划与库里的设置一致 → in_sync=true（页面显示"已一致"）
|
||
if data["in_sync"] != true {
|
||
t.Fatalf("一致的计划应判为已同步: %+v", data)
|
||
}
|
||
}
|
||
|
||
func TestHandleBackupScheduleHostFlagsDrift(t *testing.T) {
|
||
host := &fakeBackupHost{
|
||
available: true, target: "weknora-test",
|
||
show: backuphost.ShowResult{ScheduleEnabled: true, ScheduleTime: "05:00", KeepDays: 7, KeepMonths: 3, ScriptPresent: true},
|
||
}
|
||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||
recorder := httptest.NewRecorder()
|
||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||
data := decodeData(t, recorder)
|
||
if data["in_sync"] != false {
|
||
t.Fatalf("主机是 05:00、设置是 03:30，应判为待下发: %+v", data)
|
||
}
|
||
if data["host_schedule_time"] != "05:00" {
|
||
t.Fatalf("应回显主机上真实时间: %+v", data)
|
||
}
|
||
}
|
||
|
||
func TestHandleBackupScheduleHostUnreachableSeparatesConnectionFromHostState(t *testing.T) {
|
||
// 现场原错：ssh connect to host 120.76.217.51 port 22 Operation timed out。
|
||
// 连接没建立 ⇒ 既不能报"脚本缺失"，也不能报"计划未下发"，只能报不可达 + 拨号地址。
|
||
host := &fakeBackupHost{
|
||
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
|
||
showErr: errors.New("目标主机 weknora-test 不可达（ssh: connect to host 120.76.217.51 port 22: Operation timed out）；未执行任何主机侧脚本，脚本与计划状态未知"),
|
||
}
|
||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||
recorder := httptest.NewRecorder()
|
||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||
data := decodeData(t, recorder)
|
||
if data["host_reachable"] != false || data["in_sync"] != false {
|
||
t.Fatalf("连不上时不能报告可达或已同步: %+v", data)
|
||
}
|
||
message := data["host_error"].(string)
|
||
if !strings.Contains(message, "不可达") || !strings.Contains(message, "100.85.111.121:22") {
|
||
t.Fatalf("应报不可达并给出真实拨号地址: %q", message)
|
||
}
|
||
if strings.Contains(message, "脚本缺失") || strings.Contains(message, "未下发") {
|
||
t.Fatalf("未观测到的脚本/计划状态不能当成结论: %q", message)
|
||
}
|
||
}
|
||
|
||
func TestHandleBackupScheduleHostReportsMissingScriptOnlyAfterConnecting(t *testing.T) {
|
||
// 真正的脚本缺失：ssh 已连上，主机回的是脚本不存在 ⇒ 这时才允许说脚本缺失。
|
||
host := &fakeBackupHost{
|
||
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
|
||
showErr: errors.New("已连上 weknora-test，但主机侧命令失败: bash: /opt/weknora-test/install_backup_schedule.sh: No such file or directory"),
|
||
}
|
||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||
recorder := httptest.NewRecorder()
|
||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||
data := decodeData(t, recorder)
|
||
message := data["host_error"].(string)
|
||
if !strings.Contains(message, "已连上") || !strings.Contains(message, "No such file") {
|
||
t.Fatalf("已连接但脚本缺失时应原样带回主机原因: %q", message)
|
||
}
|
||
if strings.Contains(message, "不可达") {
|
||
t.Fatalf("已连上就不能说不可达: %q", message)
|
||
}
|
||
}
|
||
|
||
func TestHandleBackupScheduleHostUnconfiguredDegrades(t *testing.T) {
|
||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, &fakeBackupHost{available: false})
|
||
recorder := httptest.NewRecorder()
|
||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||
if recorder.Code != http.StatusOK {
|
||
t.Fatalf("未接线应降级为 200（页面显示未接线），得到 %d", recorder.Code)
|
||
}
|
||
data := decodeData(t, recorder)
|
||
if data["host_configured"] != false || data["in_sync"] != false {
|
||
t.Fatalf("未接线应明确报告: %+v", data)
|
||
}
|
||
if !strings.Contains(data["host_error"].(string), "未挂载") {
|
||
t.Fatalf("应说明缺少主机凭据: %+v", data["host_error"])
|
||
}
|
||
}
|
||
|
||
func TestHandleSaveBackupScheduleRejectsInvalidAndDoesNotTouchHost(t *testing.T) {
|
||
store := &fakeBackupScheduleStore{schedule: goldenSchedule}
|
||
host := &fakeBackupHost{available: true, target: "weknora-test"}
|
||
config := backupConfig(store, host)
|
||
body := `{"schedule_time":"25:99","retention_days":0,"retention_months":3,"enabled":true}`
|
||
recorder := httptest.NewRecorder()
|
||
config.handleSaveBackupSchedule(recorder, httptest.NewRequest(http.MethodPut, "/api/v1/backup-schedule", strings.NewReader(body)))
|
||
if recorder.Code != http.StatusBadRequest {
|
||
t.Fatalf("非法计划应 400，得到 %d（%s）", recorder.Code, recorder.Body.String())
|
||
}
|
||
if !strings.Contains(recorder.Body.String(), "定时时间必须是 HH:MM") {
|
||
t.Fatalf("应给出中文原因: %s", recorder.Body.String())
|
||
}
|
||
if len(store.saved) != 0 {
|
||
t.Fatal("非法计划不应写库")
|
||
}
|
||
if host.applied != 0 {
|
||
t.Fatal("非法计划不应下发到主机")
|
||
}
|
||
}
|
||
|
||
func TestHandleSaveBackupScheduleAppliesToHost(t *testing.T) {
|
||
store := &fakeBackupScheduleStore{schedule: governance.BackupSchedule{Enabled: false, ScheduleTime: "03:30", RetentionDays: 7, RetentionMonths: 3}}
|
||
host := &fakeBackupHost{
|
||
available: true, target: "weknora-test",
|
||
apply: backuphost.ApplyResult{Enabled: true, Time: "04:15", Days: 14, Months: 6, NextRun: "2026-10-10T04:15:00+0800", CronFile: "/etc/cron.d/kb-logical-backup"},
|
||
show: backuphost.ShowResult{ScheduleEnabled: true, ScheduleTime: "04:15", KeepDays: 14, KeepMonths: 6, ScriptPresent: true},
|
||
}
|
||
config := backupConfig(store, host)
|
||
body := `{"enabled":true,"schedule_time":"04:15","retention_days":14,"retention_months":6}`
|
||
recorder := httptest.NewRecorder()
|
||
config.handleSaveBackupSchedule(recorder, httptest.NewRequest(http.MethodPut, "/api/v1/backup-schedule", strings.NewReader(body)))
|
||
if recorder.Code != http.StatusOK {
|
||
t.Fatalf("期望 200，得到 %d（%s）", recorder.Code, recorder.Body.String())
|
||
}
|
||
if len(store.saved) != 1 || store.saved[0].ScheduleTime != "04:15" || !store.saved[0].Enabled {
|
||
t.Fatalf("应把新计划写库: %+v", store.saved)
|
||
}
|
||
if host.applied != 1 {
|
||
t.Fatalf("默认应下发到主机，实际 %d 次", host.applied)
|
||
}
|
||
if len(store.appliedAt) != 1 || !strings.Contains(store.appliedAt[0], "每天 04:15") {
|
||
t.Fatalf("应记录下发摘要: %+v", store.appliedAt)
|
||
}
|
||
var payload struct {
|
||
Applied bool `json:"applied"`
|
||
Data map[string]any `json:"data"`
|
||
}
|
||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !payload.Applied || payload.Data["in_sync"] != true {
|
||
t.Fatalf("下发后应报告 applied=true 且已一致: %+v", payload)
|
||
}
|
||
}
|
||
|
||
func TestHandleSaveBackupScheduleWithoutHostCredentials(t *testing.T) {
|
||
store := &fakeBackupScheduleStore{schedule: goldenSchedule}
|
||
host := &fakeBackupHost{available: false}
|
||
config := backupConfig(store, host)
|
||
recorder := httptest.NewRecorder()
|
||
config.handleSaveBackupSchedule(recorder, httptest.NewRequest(http.MethodPut, "/api/v1/backup-schedule",
|
||
strings.NewReader(`{"schedule_time":"04:15","retention_days":14,"retention_months":6,"enabled":true}`)))
|
||
if recorder.Code != http.StatusOK {
|
||
t.Fatalf("未接线时保存仍应成功（只是不下发），得到 %d", recorder.Code)
|
||
}
|
||
var payload struct {
|
||
Applied bool `json:"applied"`
|
||
}
|
||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if payload.Applied {
|
||
t.Fatal("未接线不应报告已下发")
|
||
}
|
||
if host.applied != 0 {
|
||
t.Fatal("未接线不应调用主机")
|
||
}
|
||
if len(store.saved) != 1 {
|
||
t.Fatal("计划仍应写库")
|
||
}
|
||
}
|
||
|
||
func TestHandleApplyBackupSchedulePushesCurrentSettings(t *testing.T) {
|
||
store := &fakeBackupScheduleStore{schedule: goldenSchedule}
|
||
host := &fakeBackupHost{available: true, target: "weknora-test",
|
||
apply: backuphost.ApplyResult{Enabled: true, Time: "03:30", Days: 7, Months: 3}}
|
||
config := backupConfig(store, host)
|
||
recorder := httptest.NewRecorder()
|
||
config.handleApplyBackupSchedule(recorder, httptest.NewRequest(http.MethodPost, "/api/v1/backup-schedule/apply", nil))
|
||
if recorder.Code != http.StatusOK || host.applied != 1 {
|
||
t.Fatalf("应重新下发一次: code=%d applied=%d", recorder.Code, host.applied)
|
||
}
|
||
if len(store.saved) != 0 {
|
||
t.Fatal("重新下发不应改库里的设置")
|
||
}
|
||
}
|
||
|
||
func TestHandleRunBackupNowIsAsyncAndSingleFlight(t *testing.T) {
|
||
store := &fakeBackupScheduleStore{schedule: goldenSchedule}
|
||
host := &fakeBackupHost{available: true, target: "weknora-test",
|
||
lastRun: backuphost.RunResult{Stamp: "20261009T060000Z", DurationSeconds: 34}}
|
||
config := backupConfig(store, host)
|
||
// 先占住在途标记：重复点击不能跑第二次整库 dump。
|
||
config.BackupRuns.Start(config.Now())
|
||
recorder := httptest.NewRecorder()
|
||
config.handleRunBackupNow(recorder, httptest.NewRequest(http.MethodPost, "/api/v1/backup-schedule/run", nil))
|
||
if recorder.Code != http.StatusAccepted {
|
||
t.Fatalf("期望 202（异步），得到 %d", recorder.Code)
|
||
}
|
||
data := decodeData(t, recorder)
|
||
if data["started"] != false || data["running"] != true {
|
||
t.Fatalf("已在跑时不应重复启动: %+v", data)
|
||
}
|
||
if host.runs != 0 {
|
||
t.Fatal("已被占住时不应触发主机执行")
|
||
}
|
||
}
|
||
|
||
func TestHandleRunBackupNowStartsAndFinishesTracking(t *testing.T) {
|
||
host := &fakeBackupHost{available: true, target: "weknora-test",
|
||
lastRun: backuphost.RunResult{Stamp: "20261009T060000Z", DurationSeconds: 34}}
|
||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||
recorder := httptest.NewRecorder()
|
||
config.handleRunBackupNow(recorder, httptest.NewRequest(http.MethodPost, "/api/v1/backup-schedule/run", nil))
|
||
if recorder.Code != http.StatusAccepted {
|
||
t.Fatalf("期望 202，得到 %d", recorder.Code)
|
||
}
|
||
if data := decodeData(t, recorder); data["started"] != true {
|
||
t.Fatalf("应报告已启动: %+v", data)
|
||
}
|
||
// 后台 goroutine 完成（假 RunNow 立即返回）。
|
||
deadline := time.Now().Add(2 * time.Second)
|
||
for time.Now().Before(deadline) {
|
||
if running, _, _, _ := config.BackupRuns.snapshot(); !running {
|
||
if host.runs != 1 {
|
||
t.Fatalf("应恰好执行一次，实际 %d", host.runs)
|
||
}
|
||
return
|
||
}
|
||
time.Sleep(10 * time.Millisecond)
|
||
}
|
||
t.Fatal("在途标记未在预期时间内结束")
|
||
}
|
||
|
||
func TestHandleRunBackupNowWithoutHostCredentials(t *testing.T) {
|
||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, &fakeBackupHost{available: false})
|
||
recorder := httptest.NewRecorder()
|
||
config.handleRunBackupNow(recorder, httptest.NewRequest(http.MethodPost, "/api/v1/backup-schedule/run", nil))
|
||
if recorder.Code != http.StatusServiceUnavailable {
|
||
t.Fatalf("未接线应 503，得到 %d", recorder.Code)
|
||
}
|
||
}
|
||
|
||
// 写操作必须是管理员：计划会落到数据面主机的 cron（能删数据、能占满磁盘）。
|
||
func TestBackupScheduleRoleClassification(t *testing.T) {
|
||
cases := []struct {
|
||
method, path, want string
|
||
}{
|
||
{http.MethodGet, "/api/v1/backup-schedule", roleViewer},
|
||
{http.MethodGet, "/api/v1/backup-schedule/host", roleViewer},
|
||
{http.MethodPut, "/api/v1/backup-schedule", roleAdmin},
|
||
{http.MethodPost, "/api/v1/backup-schedule/apply", roleAdmin},
|
||
{http.MethodPost, "/api/v1/backup-schedule/run", roleAdmin},
|
||
}
|
||
for _, item := range cases {
|
||
if got := classifyRole(item.method, item.path); got != item.want {
|
||
t.Errorf("%s %s: 期望 %s，得到 %s", item.method, item.path, item.want, got)
|
||
}
|
||
}
|
||
}
|