姚章浩
5e1482e349
build / build (push) Successful in 1m40s
根因：生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关， 页面没有任何入口去真跑一次恢复；CLI（librarydrill）又不是 worker 能复用的实现， 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动： - 共享实现 internal/libraryverify（维护隔离→捕获→基线校验→canary→恢复→复核→关窗口）， cmd/librarydrill 改为薄封装，CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业：kind + VerifyStages(capture/verify/canary/restore/finalize)， 与 sync/restore/baseline 共用同一把写互斥；迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications（按 target_id 绑定，换库/索引/桶旧证据自动失效， 同 target+scope+job 幂等）。 - worker：admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现，阶段实时写 library_job_stages，结束后写真实终态；通过写 passed 证据，失败写 failed+原因（不伪成功）； 中断接管按同一作业 id 收掉维护窗口。 - API：POST/GET /api/v1/library/verifications（管理员；闸口关闭/维护中拒绝；返回完整 整库目标、最近证据、失败原因），OpenAPI 同步。 - 闸口：同步/恢复/续作的证据判断改为读当前目标证据；历史 env 仅在证据缺失时兜底， 证据存在时以证据为准。 - 页面：生产保护与恢复面板加「恢复验证」块（完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询），作业类型补 verify 文案。 测试：make verify 16/16 全绿（go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定）； 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
352 lines
14 KiB
Go
352 lines
14 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"net/http"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/backuphost"
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// BackupScheduleStore 持久化「系统设置 → 存储与备份」里的整库备份计划（migration 000045）。
|
||
type BackupScheduleStore interface {
|
||
LoadBackupSchedule(context.Context) (governance.BackupSchedule, error)
|
||
SaveBackupSchedule(context.Context, governance.BackupSchedule, string, time.Time) error
|
||
MarkBackupScheduleApplied(context.Context, string, time.Time) error
|
||
}
|
||
|
||
// BackupHost 在目标库所在主机上读回/下发计划并触发一次备份。未接线（Available=false）时
|
||
// 页面降级为"主机凭据未挂载"，而不是报 500。
|
||
type BackupHost interface {
|
||
Available() bool
|
||
Target() string
|
||
// Endpoint 是挂载的 ssh 配置里该别名真实拨号的 host:port；页面据此证明
|
||
// "连的是哪条通道"，而不是只显示一个别名。
|
||
Endpoint() string
|
||
Show(context.Context) (backuphost.ShowResult, error)
|
||
Apply(context.Context, bool, string, int, int) (backuphost.ApplyResult, error)
|
||
RunNow(context.Context) (backuphost.RunResult, error)
|
||
LastRun(context.Context) (backuphost.RunResult, error)
|
||
}
|
||
|
||
// BackupRunTracker 记录"立即备份一次"的在途状态：避免重复点击跑两次全库 dump，
|
||
// 并让页面轮询到"正在备份"而不是一个被 WriteTimeout 截断的响应。
|
||
// 进程重启会丢掉在途标记，最终事实以主机上的 MANIFEST.json 为准。
|
||
type BackupRunTracker struct {
|
||
mutex sync.Mutex
|
||
running bool
|
||
startedAt time.Time
|
||
finishedAt time.Time
|
||
lastError string
|
||
}
|
||
|
||
// Start 标记开始；已在跑时返回 false（调用方据此不重复触发）。
|
||
func (tracker *BackupRunTracker) Start(now time.Time) bool {
|
||
tracker.mutex.Lock()
|
||
defer tracker.mutex.Unlock()
|
||
if tracker.running {
|
||
return false
|
||
}
|
||
tracker.running = true
|
||
tracker.startedAt = now
|
||
tracker.lastError = ""
|
||
return true
|
||
}
|
||
|
||
// Finish 结束一次运行并记录错误（空字符串表示成功）。
|
||
func (tracker *BackupRunTracker) Finish(errText string, now time.Time) {
|
||
tracker.mutex.Lock()
|
||
defer tracker.mutex.Unlock()
|
||
tracker.running = false
|
||
tracker.finishedAt = now
|
||
tracker.lastError = errText
|
||
}
|
||
|
||
func (tracker *BackupRunTracker) snapshot() (bool, time.Time, time.Time, string) {
|
||
tracker.mutex.Lock()
|
||
defer tracker.mutex.Unlock()
|
||
return tracker.running, tracker.startedAt, tracker.finishedAt, tracker.lastError
|
||
}
|
||
|
||
type backupScheduleView struct {
|
||
Enabled bool `json:"enabled"`
|
||
ScheduleTime string `json:"schedule_time"`
|
||
RetentionDays int `json:"retention_days"`
|
||
RetentionMonths int `json:"retention_months"`
|
||
UpdatedBy string `json:"updated_by"`
|
||
UpdatedAt string `json:"updated_at"`
|
||
// LastApplied* 记录最后一次成功下发到主机的结果（页面"当前主机上的计划"）。
|
||
LastAppliedAt string `json:"last_applied_at"`
|
||
LastAppliedSummary string `json:"last_applied_summary"`
|
||
// HostConfigured/HostReachable 区分"没接线"与"接线了但连不上"。
|
||
HostConfigured bool `json:"host_configured"`
|
||
HostTarget string `json:"host_target"`
|
||
// HostEndpoint 是这条别名真实拨号的 host:port（来自挂载的 ssh 配置）。
|
||
// 连接失败时它就是"到底往哪连"的唯一证据，不能只给一个别名让人猜。
|
||
HostEndpoint string `json:"host_endpoint"`
|
||
HostReachable bool `json:"host_reachable"`
|
||
HostError string `json:"host_error"`
|
||
|
||
HostScheduleEnabled bool `json:"host_schedule_enabled"`
|
||
HostScheduleTime string `json:"host_schedule_time"`
|
||
HostKeepDays int `json:"host_keep_days"`
|
||
HostKeepMonths int `json:"host_keep_months"`
|
||
HostCronLines []string `json:"host_cron_lines"`
|
||
HostCronService string `json:"host_cron_service"`
|
||
HostScriptPresent bool `json:"host_script_present"`
|
||
|
||
LastRunStamp string `json:"last_run_stamp"`
|
||
LastRunCreatedAt string `json:"last_run_created_at"`
|
||
LastRunDurationSeconds int `json:"last_run_duration_seconds"`
|
||
LastRunTotalBytes int64 `json:"last_run_total_bytes"`
|
||
LastRunDatabases []string `json:"last_run_databases"`
|
||
LastRunFailedDatabases []string `json:"last_run_failed_databases"`
|
||
LastRunError string `json:"last_run_error"`
|
||
|
||
// InSync 表示库里的计划与主机上生效的计划一致（不一致时页面提示"待下发"）。
|
||
InSync bool `json:"in_sync"`
|
||
// Applied/ApplyError 是本次请求是否成功下发的问题说明（保存并下发、重新下发、立即备份都用）。
|
||
Applied bool `json:"applied"`
|
||
ApplyError string `json:"apply_error"`
|
||
// Running 表示"立即备份一次"正在执行。
|
||
Running bool `json:"running"`
|
||
RunningStartedAt string `json:"running_started_at"`
|
||
RunLastError string `json:"run_last_error"`
|
||
}
|
||
|
||
// handleBackupSchedule 只读返回库里的计划（不探测主机，避免主机不可达时拖慢页面）。
|
||
func (config Config) handleBackupSchedule(w http.ResponseWriter, r *http.Request) {
|
||
view, err := config.backupSchedulePayload(r.Context(), false)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load backup schedule"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusOK, map[string]any{"data": view})
|
||
}
|
||
|
||
// handleBackupScheduleHost 探测目标主机：当前 cron 计划 + 最近一次备份结果。
|
||
func (config Config) handleBackupScheduleHost(w http.ResponseWriter, r *http.Request) {
|
||
view, err := config.backupSchedulePayload(r.Context(), true)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load backup schedule"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusOK, map[string]any{"data": view})
|
||
}
|
||
|
||
// handleSaveBackupSchedule 保存计划；body 里 apply（默认 true）为真时一并下发到主机。
|
||
func (config Config) handleSaveBackupSchedule(w http.ResponseWriter, r *http.Request) {
|
||
var input struct {
|
||
Enabled *bool `json:"enabled"`
|
||
ScheduleTime string `json:"schedule_time"`
|
||
RetentionDays *int `json:"retention_days"`
|
||
RetentionMonths *int `json:"retention_months"`
|
||
Apply *bool `json:"apply"`
|
||
}
|
||
if err := decodeStrictJSON(r, &input); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||
return
|
||
}
|
||
current, err := config.BackupSchedule.LoadBackupSchedule(r.Context())
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load backup schedule"})
|
||
return
|
||
}
|
||
next := current
|
||
if input.Enabled != nil {
|
||
next.Enabled = *input.Enabled
|
||
}
|
||
if strings.TrimSpace(input.ScheduleTime) != "" {
|
||
next.ScheduleTime = strings.TrimSpace(input.ScheduleTime)
|
||
}
|
||
if input.RetentionDays != nil {
|
||
next.RetentionDays = *input.RetentionDays
|
||
}
|
||
if input.RetentionMonths != nil {
|
||
next.RetentionMonths = *input.RetentionMonths
|
||
}
|
||
next = next.Normalize()
|
||
if err := next.Validate(); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||
return
|
||
}
|
||
if err := config.BackupSchedule.SaveBackupSchedule(r.Context(), next, actorFor(r), config.Now().UTC()); err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not save backup schedule"})
|
||
return
|
||
}
|
||
// 先下发、再组装投影：否则页面保存后那几行还是"下发前"的主机状态，看着像没生效。
|
||
apply := input.Apply == nil || *input.Apply
|
||
problem, applied := "", false
|
||
if apply && config.BackupHost.Available() {
|
||
problem, applied = config.applyBackupSchedule(r.Context(), next)
|
||
}
|
||
view, err := config.backupSchedulePayload(r.Context(), true)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load backup schedule"})
|
||
return
|
||
}
|
||
view.Applied = applied
|
||
view.ApplyError = problem
|
||
writeJSON(w, http.StatusOK, map[string]any{"data": view, "applied": applied, "apply_error": problem})
|
||
}
|
||
|
||
// handleApplyBackupSchedule 不改设置，只把库里的计划重新下发一次。
|
||
func (config Config) handleApplyBackupSchedule(w http.ResponseWriter, r *http.Request) {
|
||
schedule, err := config.BackupSchedule.LoadBackupSchedule(r.Context())
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load backup schedule"})
|
||
return
|
||
}
|
||
if err := schedule.Validate(); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||
return
|
||
}
|
||
problem, applied := config.applyBackupSchedule(r.Context(), schedule)
|
||
view, err := config.backupSchedulePayload(r.Context(), true)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not load backup schedule"})
|
||
return
|
||
}
|
||
view.Applied = applied
|
||
view.ApplyError = problem
|
||
writeJSON(w, http.StatusOK, map[string]any{"data": view, "applied": applied, "apply_error": problem})
|
||
}
|
||
|
||
// handleRunBackupNow 触发一次整库备份。全库 dump 约 33 秒，超过 http 写超时，因此异步：
|
||
// 立即返回 202，页面轮询 GET /api/v1/backup-schedule/host 看到 running 与新的最近一次结果。
|
||
func (config Config) handleRunBackupNow(w http.ResponseWriter, r *http.Request) {
|
||
if !config.BackupHost.Available() {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{
|
||
"error": "目标主机凭据未挂载，无法立即备份（见 BACKUP_HOST_SSH_CONFIG）"})
|
||
return
|
||
}
|
||
now := config.Now().UTC()
|
||
if !config.BackupRuns.Start(now) {
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"data": map[string]any{"started": false, "running": true}})
|
||
return
|
||
}
|
||
go func() {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||
defer cancel()
|
||
_, err := config.BackupHost.RunNow(ctx)
|
||
message := ""
|
||
if err != nil {
|
||
message = err.Error()
|
||
}
|
||
config.BackupRuns.Finish(message, time.Now().UTC())
|
||
}()
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"data": map[string]any{"started": true, "running": true}})
|
||
}
|
||
|
||
// applyBackupSchedule 把计划下发到目标主机，并把结果摘要写回库里（页面显示"最后下发"）。
|
||
// 返回 (错误说明, 是否成功)。
|
||
func (config Config) applyBackupSchedule(ctx context.Context, schedule governance.BackupSchedule) (string, bool) {
|
||
result, err := config.BackupHost.Apply(ctx, schedule.Enabled, schedule.ScheduleTime,
|
||
schedule.RetentionDays, schedule.RetentionMonths)
|
||
if err != nil {
|
||
return err.Error(), false
|
||
}
|
||
summary := scheduleAppliedSummary(result)
|
||
if err := config.BackupSchedule.MarkBackupScheduleApplied(ctx, summary, config.Now().UTC()); err != nil {
|
||
return "计划已下发，但结果摘要未能写回：" + err.Error(), true
|
||
}
|
||
return "", true
|
||
}
|
||
|
||
func scheduleAppliedSummary(result backuphost.ApplyResult) string {
|
||
if !result.Enabled {
|
||
return "已关闭定时（已有备份保留）"
|
||
}
|
||
parts := []string{"每天 " + result.Time, "保留 " + itoa(result.Days) + " 天",
|
||
"每月归档 " + itoa(result.Months) + " 份"}
|
||
if result.NextRun != "" {
|
||
parts = append(parts, "下次 "+result.NextRun)
|
||
}
|
||
return strings.Join(parts, " · ")
|
||
}
|
||
|
||
// backupSchedulePayload 组装页面投影。probeHost=true 时探测主机（SSH 只读，短超时）。
|
||
func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool) (backupScheduleView, error) {
|
||
schedule, err := config.BackupSchedule.LoadBackupSchedule(ctx)
|
||
if err != nil {
|
||
return backupScheduleView{}, err
|
||
}
|
||
view := backupScheduleView{
|
||
Enabled: schedule.Enabled, ScheduleTime: schedule.ScheduleTime,
|
||
RetentionDays: schedule.RetentionDays, RetentionMonths: schedule.RetentionMonths,
|
||
UpdatedBy: schedule.UpdatedBy, UpdatedAt: formatTime(schedule.UpdatedAt),
|
||
LastAppliedSummary: schedule.LastAppliedSummary,
|
||
InSync: true,
|
||
}
|
||
if schedule.LastAppliedAt != nil {
|
||
view.LastAppliedAt = formatTime(*schedule.LastAppliedAt)
|
||
}
|
||
view.HostConfigured = config.BackupHost.Available()
|
||
view.HostTarget = config.BackupHost.Target()
|
||
view.HostEndpoint = config.BackupHost.Endpoint()
|
||
if !view.HostConfigured {
|
||
view.InSync = false
|
||
view.HostError = "目标主机凭据未挂载（BACKUP_HOST_SSH_CONFIG / BACKUP_HOST_TARGET）"
|
||
return view, nil
|
||
}
|
||
if !probeHost {
|
||
return view, nil
|
||
}
|
||
show, err := config.BackupHost.Show(ctx)
|
||
if err != nil {
|
||
view.HostReachable = false
|
||
view.HostError = err.Error()
|
||
if view.HostEndpoint != "" {
|
||
view.HostError += "（拨号地址 " + view.HostEndpoint + "）"
|
||
}
|
||
view.InSync = false
|
||
} else {
|
||
view.HostReachable = true
|
||
view.HostScheduleEnabled = show.ScheduleEnabled
|
||
view.HostScheduleTime = show.ScheduleTime
|
||
view.HostKeepDays = show.KeepDays
|
||
view.HostKeepMonths = show.KeepMonths
|
||
view.HostCronLines = show.CronLines
|
||
view.HostCronService = show.CronService
|
||
view.HostScriptPresent = show.ScriptPresent
|
||
hostSchedule := governance.BackupSchedule{
|
||
Enabled: show.ScheduleEnabled, ScheduleTime: show.ScheduleTime,
|
||
RetentionDays: show.KeepDays, RetentionMonths: show.KeepMonths,
|
||
}
|
||
view.InSync = show.ScriptPresent && schedule.SameHostPlan(hostSchedule)
|
||
}
|
||
lastRun, runErr := config.BackupHost.LastRun(ctx)
|
||
if runErr != nil {
|
||
view.LastRunError = runErr.Error()
|
||
} else {
|
||
view.LastRunStamp = lastRun.Stamp
|
||
view.LastRunCreatedAt = lastRun.CreatedAt
|
||
view.LastRunDurationSeconds = lastRun.DurationSeconds
|
||
view.LastRunTotalBytes = lastRun.TotalBytes
|
||
view.LastRunDatabases = lastRun.Databases
|
||
view.LastRunFailedDatabases = lastRun.FailedDatabases
|
||
}
|
||
running, startedAt, _, lastError := config.BackupRuns.snapshot()
|
||
view.Running = running
|
||
view.RunLastError = lastError
|
||
if running {
|
||
view.RunningStartedAt = formatTime(startedAt)
|
||
}
|
||
return view, nil
|
||
}
|
||
|
||
func formatTime(value time.Time) string {
|
||
if value.IsZero() {
|
||
return ""
|
||
}
|
||
return value.UTC().Format(time.RFC3339)
|
||
}
|
||
|
||
func itoa(value int) string {
|
||
return strconv.Itoa(value)
|
||
}
|