Files
knowledge-governance/internal/backuphost/runner_test.go
T
姚章浩 5e1482e349
build / build (push) Successful in 1m40s
feat(library): 恢复验证成为可页面发起的真实作业
根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关,
页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现,
所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。

改动:
- 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口),
  cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。
- verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize),
  与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、
  新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效,
  同 target+scope+job 幂等)。
- worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写
  library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功);
  中断接管按同一作业 id 收掉维护窗口。
- API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整
  整库目标、最近证据、失败原因),OpenAPI 同步。
- 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底,
  证据存在时以证据为准。
- 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、
  管理员按钮、5s 轮询),作业类型补 verify 文案。

测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定);
新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
2026-11-03 10:14:08 +08:00

248 lines
9.9 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package backuphost
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func testRunner(exec func(ctx context.Context, timeout time.Duration, remote string) (string, error)) *Runner {
runner := New(Config{
SSHConfig: "/run/backup-host/ssh_config",
Target: "weknora-test",
ScriptDir: "/opt/weknora-test",
Root: "/srv/kb-backups",
})
if exec != nil {
runner.exec = exec
}
return runner
}
func TestRunnerUnavailableWithoutCredentials(t *testing.T) {
runner := New(Config{})
if runner.Available() {
t.Fatal("没有凭据时不应判定为已接线")
}
for name, call := range map[string]func() error{
"Show": func() error { _, err := runner.Show(context.Background()); return err },
"Apply": func() error { _, err := runner.Apply(context.Background(), true, "03:30", 7, 3); return err },
"RunNow": func() error { _, err := runner.RunNow(context.Background()); return err },
} {
if err := call(); !errors.Is(err, ErrNotConfigured) {
t.Errorf("%s: 期望 ErrNotConfigured,得到 %v", name, err)
}
}
}
// 只给一半参数(缺 Target)也算未接线,避免"看起来接上了"。
func TestRunnerPartialConfigIsNotAvailable(t *testing.T) {
runner := New(Config{SSHConfig: "/run/backup-host/ssh_config"})
if runner.Available() {
t.Fatal("缺少 Target 时不应判定为已接线")
}
}
func TestRunnerApplyBuildsCommandAndParsesResult(t *testing.T) {
var seen string
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
seen = remote
return "计划已下发:每天 03:30(保留 7 天,另按自然月保留 3 份)\n" +
"SCHEDULE_APPLIED enabled=1 time=03:30 days=7 months=3 next=2026-10-10T03:30:00+0800 cron=/etc/cron.d/kb-logical-backup\n", nil
})
result, err := runner.Apply(context.Background(), true, "03:30", 7, 3)
if err != nil {
t.Fatal(err)
}
if seen != "bash /opt/weknora-test/install_backup_schedule.sh --time 03:30 --days 7 --months 3" {
t.Fatalf("下发的远端命令不对: %q", seen)
}
if !result.Enabled || result.Time != "03:30" || result.Days != 7 || result.Months != 3 {
t.Fatalf("结果解析不对: %+v", result)
}
if result.CronFile != "/etc/cron.d/kb-logical-backup" || result.NextRun != "2026-10-10T03:30:00+0800" {
t.Fatalf("cron/下次执行解析不对: %+v", result)
}
}
func TestRunnerApplyDisable(t *testing.T) {
var seen string
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
seen = remote
return "已关闭定时备份(已生成的备份保留)。\nSCHEDULE_APPLIED enabled=0\n", nil
})
result, err := runner.Apply(context.Background(), false, "03:30", 7, 3)
if err != nil {
t.Fatal(err)
}
if seen != "bash /opt/weknora-test/install_backup_schedule.sh --disable" {
t.Fatalf("关闭定时应调用 --disable: %q", seen)
}
if result.Enabled {
t.Fatal("关闭后 enabled 应为 false")
}
}
// 主机没按预期输出 SCHEDULE_APPLIED(例如脚本版本旧)时必须报错,绝不能默默当成功。
func TestRunnerApplyFailsWithoutMarker(t *testing.T) {
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
return "bash: /opt/weknora-test/install_backup_schedule.sh: No such file or directory\n", nil
})
if _, err := runner.Apply(context.Background(), true, "03:30", 7, 3); err == nil {
t.Fatal("缺少 SCHEDULE_APPLIED 应报错")
}
}
func TestRunnerShowParsesHostState(t *testing.T) {
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
return `备份脚本: /opt/weknora-test/logical-backup.sh (存在)
计划参数: KB_BACKUP_SCHEDULE_ENABLED=1 KB_BACKUP_SCHEDULE_TIME=03:30 KB_BACKUP_KEEP_DAYS=7 KB_BACKUP_KEEP_MONTHS=3 KB_BACKUP_SCRIPT=/opt/weknora-test/logical-backup.sh KB_BACKUP_LOG=/srv/kb-backups/logs/logical-backup.log
cron 条目:
30 03 * * * root /bin/bash /opt/weknora-test/logical-backup.sh >> /srv/kb-backups/logs/logical-backup.log 2>&1
cron 服务: active
备份根目录: /srv/kb-backups(758M 已用)
`, nil
})
show, err := runner.Show(context.Background())
if err != nil {
t.Fatal(err)
}
if !show.ScheduleEnabled || show.ScheduleTime != "03:30" || show.KeepDays != 7 || show.KeepMonths != 3 {
t.Fatalf("计划参数解析不对: %+v", show)
}
if len(show.CronLines) != 1 || !strings.Contains(show.CronLines[0], "logical-backup.sh") {
t.Fatalf("cron 条目解析不对: %+v", show.CronLines)
}
if show.CronService != "active" || !show.ScriptPresent {
t.Fatalf("服务/脚本状态解析不对: %+v", show)
}
}
// 未配置计划时主机输出"未配置/未安装",解析不能崩、也不能谎报为已启用。
func TestRunnerShowHandlesUnconfiguredHost(t *testing.T) {
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
return "备份脚本: /opt/weknora-test/logical-backup.sh (存在)\n计划参数: 未配置\ncron 条目: 未安装(定时关闭)\ncron 服务: active\n", nil
})
show, err := runner.Show(context.Background())
if err != nil {
t.Fatal(err)
}
if show.ScheduleEnabled || show.ScheduleTime != "" || len(show.CronLines) != 0 {
t.Fatalf("未配置主机不应解析成已启用: %+v", show)
}
}
func TestRunnerLastRunParsesManifest(t *testing.T) {
var seen string
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
seen = remote
return `STAMP=20261009T052909Z
{
"kind": "rds-logical-backup",
"created_at": "20261009T052909Z",
"duration_seconds": 33,
"databases": ["dobotai_prod", "weknora_test"],
"failed_databases": ["rdsadmin"],
"total_bytes": 778502144
}
`, nil
})
lastRun, err := runner.LastRun(context.Background())
if err != nil {
t.Fatal(err)
}
if !strings.Contains(seen, "/srv/kb-backups/rds-logical") {
t.Fatalf("应读取契约路径下的 rds-logical: %q", seen)
}
if lastRun.Stamp != "20261009T052909Z" || lastRun.DurationSeconds != 33 || lastRun.TotalBytes != 778502144 {
t.Fatalf("清单解析不对: %+v", lastRun)
}
if len(lastRun.FailedDatabases) != 1 || lastRun.FailedDatabases[0] != "rdsadmin" {
t.Fatalf("失败库解析不对: %+v", lastRun.FailedDatabases)
}
}
// 还没跑过备份:STAMP 为空且无清单,应得到空结果而不是错误。
func TestRunnerLastRunWithoutSnapshot(t *testing.T) {
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
return "STAMP=\n", nil
})
lastRun, err := runner.LastRun(context.Background())
if err != nil {
t.Fatal(err)
}
if lastRun.Stamp != "" || lastRun.DurationSeconds != 0 {
t.Fatalf("无备份时应为空结果: %+v", lastRun)
}
}
// 主机侧错误(ssh 失败、脚本非零退出)必须带着输出回传,页面才能显示原因。
func TestRunnerSurfacesHostFailure(t *testing.T) {
runner := testRunner(func(context.Context, time.Duration, string) (string, error) {
return "", errors.New("ssh weknora-test 执行失败: bash: install_backup_schedule.sh: No such file")
})
if _, err := runner.Show(context.Background()); err == nil || !strings.Contains(err.Error(), "No such file") {
t.Fatalf("应把主机侧失败原因带回来,得到 %v", err)
}
}
// 现场原错:ssh: connect to host 120.76.217.51 port 22: Operation timed out。
// 连不上时脚本根本没跑,只能说"不可达";断言配对的反面:不能说脚本缺失。
func TestHostFailureSeparatesUnreachableFromRemoteCommand(t *testing.T) {
unreachable := hostFailure("weknora-test", "ssh: connect to host 120.76.217.51 port 22: Operation timed out")
if !strings.Contains(unreachable.Error(), "不可达") || !strings.Contains(unreachable.Error(), "Operation timed out") {
t.Fatalf("连接超时应报不可达并带原错: %v", unreachable)
}
if strings.Contains(unreachable.Error(), "主机侧命令失败") || strings.Contains(unreachable.Error(), "脚本缺失") {
t.Fatalf("连接失败不能报成远端命令/脚本问题: %v", unreachable)
}
remote := hostFailure("weknora-test", "bash: /opt/weknora-test/logical-backup.sh: No such file or directory")
if !strings.Contains(remote.Error(), "已连上") || !strings.Contains(remote.Error(), "No such file") {
t.Fatalf("已连接但脚本失败时应原样带回原因: %v", remote)
}
if strings.Contains(remote.Error(), "不可达") {
t.Fatalf("已连上就不能说不可达: %v", remote)
}
}
// 页面要能说清"实际拨号的地址",否则改错 ssh_config 也看不出来。
func TestRunnerEndpointReadsDialedHostPort(t *testing.T) {
path := filepath.Join(t.TempDir(), "ssh_config")
config := "Host other\n HostName 10.0.0.9\n Port 2222\n\nHost weknora-test\n HostName 100.85.111.121\n HostKeyAlias 120.76.217.51\n"
if err := os.WriteFile(path, []byte(config), 0o600); err != nil {
t.Fatal(err)
}
runner := New(Config{SSHConfig: path, Target: "weknora-test", ScriptDir: "/opt/weknora-test"})
if got := runner.Endpoint(); got != "100.85.111.121:22" {
t.Fatalf("应报真实拨号地址(默认 22),得到 %q", got)
}
if got := New(Config{SSHConfig: filepath.Join(t.TempDir(), "missing"), Target: "weknora-test"}).Endpoint(); got != "" {
t.Fatalf("配置文件不可读时应返回空,不猜地址,得到 %q", got)
}
}
func TestRunnerRunNowReadsBackLastRun(t *testing.T) {
calls := 0
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
calls++
if strings.Contains(remote, "logical-backup.sh") && !strings.Contains(remote, "ls -1t") {
return "备份目标: /srv/kb-backups/rds-logical/20261009T060000Z\ndone: /srv/kb-backups/rds-logical/20261009T060000Z\n", nil
}
return "STAMP=20261009T060000Z\n{\"duration_seconds\": 34, \"total_bytes\": 800000000, \"failed_databases\": []}\n", nil
})
run, err := runner.RunNow(context.Background())
if err != nil {
t.Fatal(err)
}
if calls != 2 {
t.Fatalf("应立即执行一次并回读一次,实际调用 %d 次", calls)
}
if run.Stamp != "20261009T060000Z" || run.DurationSeconds != 34 || run.TotalBytes != 800000000 {
t.Fatalf("立即备份结果不对: %+v", run)
}
}