5e1482e349
build / build (push) Successful in 1m40s
根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关, 页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现, 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动: - 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口), cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize), 与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效, 同 target+scope+job 幂等)。 - worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写 library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功); 中断接管按同一作业 id 收掉维护窗口。 - API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整 整库目标、最近证据、失败原因),OpenAPI 同步。 - 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底, 证据存在时以证据为准。 - 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询),作业类型补 verify 文案。 测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定); 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
248 lines
9.9 KiB
Go
248 lines
9.9 KiB
Go
package backuphost
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
func testRunner(exec func(ctx context.Context, timeout time.Duration, remote string) (string, error)) *Runner {
|
||
runner := New(Config{
|
||
SSHConfig: "/run/backup-host/ssh_config",
|
||
Target: "weknora-test",
|
||
ScriptDir: "/opt/weknora-test",
|
||
Root: "/srv/kb-backups",
|
||
})
|
||
if exec != nil {
|
||
runner.exec = exec
|
||
}
|
||
return runner
|
||
}
|
||
|
||
func TestRunnerUnavailableWithoutCredentials(t *testing.T) {
|
||
runner := New(Config{})
|
||
if runner.Available() {
|
||
t.Fatal("没有凭据时不应判定为已接线")
|
||
}
|
||
for name, call := range map[string]func() error{
|
||
"Show": func() error { _, err := runner.Show(context.Background()); return err },
|
||
"Apply": func() error { _, err := runner.Apply(context.Background(), true, "03:30", 7, 3); return err },
|
||
"RunNow": func() error { _, err := runner.RunNow(context.Background()); return err },
|
||
} {
|
||
if err := call(); !errors.Is(err, ErrNotConfigured) {
|
||
t.Errorf("%s: 期望 ErrNotConfigured,得到 %v", name, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// 只给一半参数(缺 Target)也算未接线,避免"看起来接上了"。
|
||
func TestRunnerPartialConfigIsNotAvailable(t *testing.T) {
|
||
runner := New(Config{SSHConfig: "/run/backup-host/ssh_config"})
|
||
if runner.Available() {
|
||
t.Fatal("缺少 Target 时不应判定为已接线")
|
||
}
|
||
}
|
||
|
||
func TestRunnerApplyBuildsCommandAndParsesResult(t *testing.T) {
|
||
var seen string
|
||
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
|
||
seen = remote
|
||
return "计划已下发:每天 03:30(保留 7 天,另按自然月保留 3 份)\n" +
|
||
"SCHEDULE_APPLIED enabled=1 time=03:30 days=7 months=3 next=2026-10-10T03:30:00+0800 cron=/etc/cron.d/kb-logical-backup\n", nil
|
||
})
|
||
result, err := runner.Apply(context.Background(), true, "03:30", 7, 3)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if seen != "bash /opt/weknora-test/install_backup_schedule.sh --time 03:30 --days 7 --months 3" {
|
||
t.Fatalf("下发的远端命令不对: %q", seen)
|
||
}
|
||
if !result.Enabled || result.Time != "03:30" || result.Days != 7 || result.Months != 3 {
|
||
t.Fatalf("结果解析不对: %+v", result)
|
||
}
|
||
if result.CronFile != "/etc/cron.d/kb-logical-backup" || result.NextRun != "2026-10-10T03:30:00+0800" {
|
||
t.Fatalf("cron/下次执行解析不对: %+v", result)
|
||
}
|
||
}
|
||
|
||
func TestRunnerApplyDisable(t *testing.T) {
|
||
var seen string
|
||
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
|
||
seen = remote
|
||
return "已关闭定时备份(已生成的备份保留)。\nSCHEDULE_APPLIED enabled=0\n", nil
|
||
})
|
||
result, err := runner.Apply(context.Background(), false, "03:30", 7, 3)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if seen != "bash /opt/weknora-test/install_backup_schedule.sh --disable" {
|
||
t.Fatalf("关闭定时应调用 --disable: %q", seen)
|
||
}
|
||
if result.Enabled {
|
||
t.Fatal("关闭后 enabled 应为 false")
|
||
}
|
||
}
|
||
|
||
// 主机没按预期输出 SCHEDULE_APPLIED(例如脚本版本旧)时必须报错,绝不能默默当成功。
|
||
func TestRunnerApplyFailsWithoutMarker(t *testing.T) {
|
||
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
|
||
return "bash: /opt/weknora-test/install_backup_schedule.sh: No such file or directory\n", nil
|
||
})
|
||
if _, err := runner.Apply(context.Background(), true, "03:30", 7, 3); err == nil {
|
||
t.Fatal("缺少 SCHEDULE_APPLIED 应报错")
|
||
}
|
||
}
|
||
|
||
func TestRunnerShowParsesHostState(t *testing.T) {
|
||
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
|
||
return `备份脚本: /opt/weknora-test/logical-backup.sh (存在)
|
||
计划参数: KB_BACKUP_SCHEDULE_ENABLED=1 KB_BACKUP_SCHEDULE_TIME=03:30 KB_BACKUP_KEEP_DAYS=7 KB_BACKUP_KEEP_MONTHS=3 KB_BACKUP_SCRIPT=/opt/weknora-test/logical-backup.sh KB_BACKUP_LOG=/srv/kb-backups/logs/logical-backup.log
|
||
cron 条目:
|
||
30 03 * * * root /bin/bash /opt/weknora-test/logical-backup.sh >> /srv/kb-backups/logs/logical-backup.log 2>&1
|
||
cron 服务: active
|
||
备份根目录: /srv/kb-backups(758M 已用)
|
||
`, nil
|
||
})
|
||
show, err := runner.Show(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !show.ScheduleEnabled || show.ScheduleTime != "03:30" || show.KeepDays != 7 || show.KeepMonths != 3 {
|
||
t.Fatalf("计划参数解析不对: %+v", show)
|
||
}
|
||
if len(show.CronLines) != 1 || !strings.Contains(show.CronLines[0], "logical-backup.sh") {
|
||
t.Fatalf("cron 条目解析不对: %+v", show.CronLines)
|
||
}
|
||
if show.CronService != "active" || !show.ScriptPresent {
|
||
t.Fatalf("服务/脚本状态解析不对: %+v", show)
|
||
}
|
||
}
|
||
|
||
// 未配置计划时主机输出"未配置/未安装",解析不能崩、也不能谎报为已启用。
|
||
func TestRunnerShowHandlesUnconfiguredHost(t *testing.T) {
|
||
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
|
||
return "备份脚本: /opt/weknora-test/logical-backup.sh (存在)\n计划参数: 未配置\ncron 条目: 未安装(定时关闭)\ncron 服务: active\n", nil
|
||
})
|
||
show, err := runner.Show(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if show.ScheduleEnabled || show.ScheduleTime != "" || len(show.CronLines) != 0 {
|
||
t.Fatalf("未配置主机不应解析成已启用: %+v", show)
|
||
}
|
||
}
|
||
|
||
func TestRunnerLastRunParsesManifest(t *testing.T) {
|
||
var seen string
|
||
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
|
||
seen = remote
|
||
return `STAMP=20261009T052909Z
|
||
{
|
||
"kind": "rds-logical-backup",
|
||
"created_at": "20261009T052909Z",
|
||
"duration_seconds": 33,
|
||
"databases": ["dobotai_prod", "weknora_test"],
|
||
"failed_databases": ["rdsadmin"],
|
||
"total_bytes": 778502144
|
||
}
|
||
`, nil
|
||
})
|
||
lastRun, err := runner.LastRun(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !strings.Contains(seen, "/srv/kb-backups/rds-logical") {
|
||
t.Fatalf("应读取契约路径下的 rds-logical: %q", seen)
|
||
}
|
||
if lastRun.Stamp != "20261009T052909Z" || lastRun.DurationSeconds != 33 || lastRun.TotalBytes != 778502144 {
|
||
t.Fatalf("清单解析不对: %+v", lastRun)
|
||
}
|
||
if len(lastRun.FailedDatabases) != 1 || lastRun.FailedDatabases[0] != "rdsadmin" {
|
||
t.Fatalf("失败库解析不对: %+v", lastRun.FailedDatabases)
|
||
}
|
||
}
|
||
|
||
// 还没跑过备份:STAMP 为空且无清单,应得到空结果而不是错误。
|
||
func TestRunnerLastRunWithoutSnapshot(t *testing.T) {
|
||
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
|
||
return "STAMP=\n", nil
|
||
})
|
||
lastRun, err := runner.LastRun(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if lastRun.Stamp != "" || lastRun.DurationSeconds != 0 {
|
||
t.Fatalf("无备份时应为空结果: %+v", lastRun)
|
||
}
|
||
}
|
||
|
||
// 主机侧错误(ssh 失败、脚本非零退出)必须带着输出回传,页面才能显示原因。
|
||
func TestRunnerSurfacesHostFailure(t *testing.T) {
|
||
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
|
||
return "", errors.New("ssh weknora-test 执行失败: bash: install_backup_schedule.sh: No such file")
|
||
})
|
||
if _, err := runner.Show(context.Background()); err == nil || !strings.Contains(err.Error(), "No such file") {
|
||
t.Fatalf("应把主机侧失败原因带回来,得到 %v", err)
|
||
}
|
||
}
|
||
|
||
// 现场原错:ssh: connect to host 120.76.217.51 port 22: Operation timed out。
|
||
// 连不上时脚本根本没跑,只能说"不可达";断言配对的反面:不能说脚本缺失。
|
||
func TestHostFailureSeparatesUnreachableFromRemoteCommand(t *testing.T) {
|
||
unreachable := hostFailure("weknora-test", "ssh: connect to host 120.76.217.51 port 22: Operation timed out")
|
||
if !strings.Contains(unreachable.Error(), "不可达") || !strings.Contains(unreachable.Error(), "Operation timed out") {
|
||
t.Fatalf("连接超时应报不可达并带原错: %v", unreachable)
|
||
}
|
||
if strings.Contains(unreachable.Error(), "主机侧命令失败") || strings.Contains(unreachable.Error(), "脚本缺失") {
|
||
t.Fatalf("连接失败不能报成远端命令/脚本问题: %v", unreachable)
|
||
}
|
||
remote := hostFailure("weknora-test", "bash: /opt/weknora-test/logical-backup.sh: No such file or directory")
|
||
if !strings.Contains(remote.Error(), "已连上") || !strings.Contains(remote.Error(), "No such file") {
|
||
t.Fatalf("已连接但脚本失败时应原样带回原因: %v", remote)
|
||
}
|
||
if strings.Contains(remote.Error(), "不可达") {
|
||
t.Fatalf("已连上就不能说不可达: %v", remote)
|
||
}
|
||
}
|
||
|
||
// 页面要能说清"实际拨号的地址",否则改错 ssh_config 也看不出来。
|
||
func TestRunnerEndpointReadsDialedHostPort(t *testing.T) {
|
||
path := filepath.Join(t.TempDir(), "ssh_config")
|
||
config := "Host other\n HostName 10.0.0.9\n Port 2222\n\nHost weknora-test\n HostName 100.85.111.121\n HostKeyAlias 120.76.217.51\n"
|
||
if err := os.WriteFile(path, []byte(config), 0o600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
runner := New(Config{SSHConfig: path, Target: "weknora-test", ScriptDir: "/opt/weknora-test"})
|
||
if got := runner.Endpoint(); got != "100.85.111.121:22" {
|
||
t.Fatalf("应报真实拨号地址(默认 22),得到 %q", got)
|
||
}
|
||
if got := New(Config{SSHConfig: filepath.Join(t.TempDir(), "missing"), Target: "weknora-test"}).Endpoint(); got != "" {
|
||
t.Fatalf("配置文件不可读时应返回空,不猜地址,得到 %q", got)
|
||
}
|
||
}
|
||
|
||
func TestRunnerRunNowReadsBackLastRun(t *testing.T) {
|
||
calls := 0
|
||
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
|
||
calls++
|
||
if strings.Contains(remote, "logical-backup.sh") && !strings.Contains(remote, "ls -1t") {
|
||
return "备份目标: /srv/kb-backups/rds-logical/20261009T060000Z\ndone: /srv/kb-backups/rds-logical/20261009T060000Z\n", nil
|
||
}
|
||
return "STAMP=20261009T060000Z\n{\"duration_seconds\": 34, \"total_bytes\": 800000000, \"failed_databases\": []}\n", nil
|
||
})
|
||
run, err := runner.RunNow(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if calls != 2 {
|
||
t.Fatalf("应立即执行一次并回读一次,实际调用 %d 次", calls)
|
||
}
|
||
if run.Stamp != "20261009T060000Z" || run.DurationSeconds != 34 || run.TotalBytes != 800000000 {
|
||
t.Fatalf("立即备份结果不对: %+v", run)
|
||
}
|
||
}
|