feat(library): 恢复验证成为可页面发起的真实作业
build / build (push) Successful in 1m40s

根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关,
页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现,
所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。

改动:
- 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口),
  cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。
- verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize),
  与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、
  新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效,
  同 target+scope+job 幂等)。
- worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写
  library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功);
  中断接管按同一作业 id 收掉维护窗口。
- API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整
  整库目标、最近证据、失败原因),OpenAPI 同步。
- 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底,
  证据存在时以证据为准。
- 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、
  管理员按钮、5s 轮询),作业类型补 verify 文案。

测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定);
新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
This commit is contained in:
姚章浩
2026-11-03 10:14:08 +08:00
parent 4421dd38ce
commit 5e1482e349
47 changed files with 2502 additions and 353 deletions
+87
View File
@@ -1655,6 +1655,93 @@
}
}
},
"/api/v1/library/verifications": {
"get": {
"tags": [
"Runtime"
],
"operationId": "getLibraryRestoreVerification",
"summary": "当前整库目标的恢复验证证据:verified 只由真实演练记录回答(按 target_id 绑定,换目标自动失效),并返回完整目标与失败原因",
"parameters": [
{
"name": "scope",
"in": "query",
"required": false,
"schema": {
"type": "string",
"enum": [
"internal",
"external"
]
}
}
],
"responses": {
"200": {
"description": "目标与证据:verified/reason/latest/history/target"
},
"400": {
"$ref": "#/components/responses/BadRequest"
},
"500": {
"$ref": "#/components/responses/InternalError"
},
"503": {
"description": "恢复验证存储不可用或证据读取失败"
}
}
},
"post": {
"tags": [
"Runtime"
],
"operationId": "startLibraryRestoreVerification",
"summary": "发起一次真实恢复验证(管理员):维护隔离下捕获目标 → 校验基线 → 写 canary 证明校验有效 → 按预备份恢复 → 复核,通过后才产生同步闸口认的证据",
"responses": {
"202": {
"description": "已创建 verify 作业(worker 在维护窗口内真实执行)"
},
"400": {
"$ref": "#/components/responses/BadRequest"
},
"403": {
"description": "不是管理员"
},
"409": {
"description": "目标正在维护或已有在飞写作业(同步/恢复/基线/验证互斥)"
},
"500": {
"$ref": "#/components/responses/InternalError"
},
"503": {
"description": "生产写入闸口关闭或恢复验证存储不可用"
}
},
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": false,
"required": [
"scope"
],
"properties": {
"scope": {
"type": "string",
"enum": [
"internal",
"external"
]
}
}
}
}
}
}
}
},
"/api/v1/library/snapshots/{id}": {
"get": {
"tags": [
+2
View File
@@ -232,6 +232,8 @@ func run(ctx context.Context, getenv func(string) string) error {
LibraryProductionInternalKB: config.LibraryProductionInternalKB,
LibraryProductionExternalKB: config.LibraryProductionExternalKB,
LibraryProductionTenantID: config.LibraryProductionTenantID,
LibraryProductionIndex: config.LibraryProductionIndex,
LibraryProductionS3Bucket: config.LibraryProductionS3Bucket,
LibraryRetentionVersions: config.LibraryRetentionVersions,
LibraryRetentionDays: config.LibraryRetentionDays,
BackupHost: backuphost.New(backuphost.Config{
-21
View File
@@ -1,21 +0,0 @@
package main
import (
"context"
"knowledge-governance/internal/libraryrelease"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error {
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
if err := ledger.Seal(); err != nil {
return err
}
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
return err
}
if err := mark(ctx); err != nil {
return err
}
return write(ctx)
}
+12 -203
View File
@@ -1,4 +1,4 @@
// command librarydrill 在目标实例上做一次**真实**的恢复演练:
// command librarydrill 是恢复演练的**命令行入口**,实现在 internal/libraryverify:
//
// 维护隔离(关入口/停写入者/证明排空)
// → 捕获目标整库(真预备份,落归档)
@@ -8,16 +8,14 @@
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
//
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境
// (与 worker 同一份 LIBRARY_* 配置),不进命令行、不打印。
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
// 为什么只剩薄封装:页面要能点一次"恢复验证",worker 必须在进程内跑同一份实现。
// 两份实现意味着"CLI 通过、页面失败"或反过来,演练结论就不可信了。
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境。
package main
import (
"context"
"crypto/rand"
"encoding/json"
"errors"
"flag"
"fmt"
"log"
@@ -26,35 +24,12 @@ import (
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarystack"
"knowledge-governance/internal/libraryverify"
)
type drillVerification struct {
SnapshotID string `json:"snapshot_id"`
Verified bool `json:"verified"`
Error string `json:"error,omitempty"`
At time.Time `json:"at"`
}
type drillReport struct {
Status string `json:"status"` // passed | failed | maintenance_held
Scope string `json:"scope"`
Target string `json:"target"`
SnapshotID string `json:"snapshot_id"`
Project string `json:"project"`
Counts map[string]int64 `json:"counts"`
BaselineCheck *drillVerification `json:"baseline_check"`
DriftCheck *drillVerification `json:"drift_check"` // 必须失败:证明确实能发现差异
RestoreCheck *drillVerification `json:"restore_check"`
Canary string `json:"canary"`
Failure string `json:"failure,omitempty"`
Stage string `json:"stage,omitempty"`
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at"`
Until map[string]string `json:"until,omitempty"`
}
// 报告类型直接复用共享实现:命令行与页面/worker 的证据字段完全一致。
type drillVerification = libraryverify.Verification
type drillReport = libraryverify.Report
func main() {
scope := flag.String("scope", "internal", "internal|external")
@@ -88,18 +63,7 @@ func recoverWindows() error {
if err != nil {
return fmt.Errorf("load config: %w", err)
}
manager, err := librarystack.OpenMaintenance(config)
if err != nil {
return err
}
defer manager.Close()
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
if err != nil {
return err
}
defer control.Close()
defer coordinator.Close()
if err := coordinator.Recover(context.Background()); err != nil {
if err := libraryverify.RecoverWindows(config); err != nil {
return err
}
log.Printf("maintenance windows recovered; target should be back to its original service state")
@@ -107,170 +71,15 @@ func recoverWindows() error {
}
func run(scope, jobID string) (*drillReport, error) {
if scope != "internal" && scope != "external" {
return nil, errors.New("scope must be internal or external")
}
config, err := appconfig.Load(os.Getenv)
if err != nil {
return nil, fmt.Errorf("load config: %w", err)
}
ctx := context.Background()
report := &drillReport{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
fail := func(stage string, cause error, held bool) (*drillReport, error) {
report.Stage = stage
report.Failure = cause.Error()
report.FinishedAt = time.Now().UTC()
if held {
report.Status = "maintenance_held"
}
return report, cause
}
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
if err != nil {
return fail("open_target", err, false)
}
defer target.Close()
manager, err := librarystack.OpenMaintenance(config)
if err != nil {
return fail("open_maintenance", err, false)
}
defer manager.Close()
if err := manager.VerifyTopology(ctx); err != nil {
return fail("topology", err, false)
}
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
if err != nil {
return fail("open_coordinator", err, false)
}
defer control.Close()
defer coordinator.Close()
if jobID == "" {
jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405")
jobID = fmt.Sprintf("drill-%s-%d", scope, time.Now().UTC().UnixNano())
}
var (
snapshotID string
counts map[string]int64
canary string
contentTouched bool
baseline drillVerification
drift drillVerification
restored drillVerification
)
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
start := time.Now()
current, err := target.Current(ctx)
if err != nil {
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
}
counts = current.Counts
report.Counts = counts
// 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。
backup, err := backups.Prepare(ctx, "", current)
if err != nil {
return "", errors.New("捕获目标整库失败:" + err.Error())
}
snapshotID = backup.SnapshotID
report.SnapshotID = snapshotID
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
return "", errors.New("记录预备份身份失败:" + err.Error())
}
log.Printf("drill: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
// 2) 基线校验:预备份必须与目标完全一致。
baseline = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
baseline.Error = err.Error()
report.BaselineCheck = &baseline
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
}
baseline.Verified = true
report.BaselineCheck = &baseline
// Canary is an isolated tag, including on an empty target. Never seed
// a document or change an existing row merely to make a drill possible.
var random [16]byte
if _, err := rand.Read(random[:]); err != nil {
return "", err
}
canary = fmt.Sprintf("%x", random[:])
identityValue := target.Identity()
var exists bool
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
return "", err
}
if exists {
return "", errors.New("canary identity collision; no write attempted")
}
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
report.Canary = canary
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
func(ctx context.Context) error {
if err := window.MarkContentRisk(ctx); err != nil {
return err
}
contentTouched = true
return nil
}, func(ctx context.Context) error {
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
return err
}); err != nil {
return "", fmt.Errorf("canary preparation/write failed: %w", err)
}
drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
report.DriftCheck = &drift
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
} else {
drift.Error = verifyErr.Error()
}
report.DriftCheck = &drift
log.Printf("drill: drift detected as expected (%s)", drift.Error)
// 4) 按预备份恢复。
// Restore removes only the exact new canary identity in the durable write set.
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
WriteSet: &ledger}); err != nil {
return "", errors.New("按预备份恢复失败:" + err.Error())
}
// 5) 恢复后逐类复核:必须与预备份完全一致。
restored = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
restored.Error = err.Error()
report.RestoreCheck = &restored
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
}
restored.Verified = true
report.RestoreCheck = &restored
log.Printf("drill: restored and verified snapshot=%s", snapshotID)
return snapshotID, nil
}
window, windowErr := coordinator.RunWindow(ctx, jobID,
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
func(error) bool { return !contentTouched || restored.Verified })
report.Until = map[string]string{"job": jobID, "window_state": window.State}
if windowErr != nil || window.State != "succeeded" {
cause := windowErr
if cause == nil {
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
}
if window.State == "held" {
return fail("window", cause, true)
}
return fail("window", cause, false)
}
report.Status = "passed"
report.FinishedAt = time.Now().UTC()
return report, nil
return libraryverify.Run(context.TODO(), config, scope, jobID,
func(stage, state, detail string) { log.Printf("stage=%s state=%s %s", stage, state, detail) })
}
func writeReport(path string, report *drillReport) error {
+6 -2
View File
@@ -233,8 +233,8 @@ func runLibraryReleaseAttempt(ctx context.Context, writer *libraryAttemptWriter,
if !config.ProductionReleaseRequestsEnabled {
gateReasons = append(gateReasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
if ok, reason := libraryRestoreEvidenceSatisfied(ctx, store, config, payload.Scope); !ok {
gateReasons = append(gateReasons, reason)
}
if len(gateReasons) > 0 {
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
@@ -628,6 +628,10 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore,
// 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口,
// 继续捕获/校验,而不是留下一个 held 目标等人工处理。
queueType = libraryBaselineJobType
case governance.LibraryJobVerify:
// 恢复验证同样持有维护窗口:接管时必须用同一份作业 id 收掉那个窗口,
// 否则目标会一直停在维护里而作业显示失败。
queueType = libraryVerifyJobType
}
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
"target_snapshot_id": job.TargetSnapshotID})
+5
View File
@@ -30,4 +30,9 @@ func registerLibraryReleaseHandlers(handlers map[string]jobs.Handler, store *pos
handlers[libraryBaselineJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
return runLibraryBaseline(ctx, store, config, job)
})
// 恢复验证:真实"备份→canary→恢复→复核",通过后写入与当前目标绑定的可恢复证据。
// 它是同步闸口的证据来源,所以与上面三类共用同一把执行锁与准入。
handlers[libraryVerifyJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
return runLibraryVerify(ctx, store, config, job)
})
}
+4 -2
View File
@@ -15,7 +15,8 @@ import (
func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
handlers := map[string]jobs.Handler{}
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, nil)
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType}
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
libraryBaselineJobType, libraryVerifyJobType}
if len(handlers) != len(want) {
t.Fatalf("registered %d handlers, want %d: %#v", len(handlers), len(want), handlers)
}
@@ -30,7 +31,8 @@ func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
func TestRegisterLibraryReleaseHandlersFailsClosedWhenNotAdmitted(t *testing.T) {
handlers := map[string]jobs.Handler{}
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, errNotAdmitted{})
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType} {
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
libraryBaselineJobType, libraryVerifyJobType} {
err := handlers[jobType](context.Background(), jobs.Job{})
if err == nil || !strings.Contains(err.Error(), "准入未通过") {
t.Fatalf("handler %q error = %v, want admission refusal", jobType, err)
+188
View File
@@ -0,0 +1,188 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryverify"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// libraryVerifyPayload 是"恢复验证"的队列载荷。
//
// 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方
// 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。
type libraryVerifyPayload struct {
LibraryJobID string `json:"library_job_id"`
Scope string `json:"scope"`
}
// runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。
//
// 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary
// 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。
// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。
func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryVerifyPayload
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
(payload.Scope != "internal" && payload.Scope != "external") {
return errors.New("invalid library verify payload")
}
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope,
governance.LibraryJobVerify, true); admitErr != nil {
return admitErr
}
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID,
func(runCtx context.Context, writer *libraryAttemptWriter) error {
return runLibraryVerifyAttempt(runCtx, writer, config, payload)
})
}
func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
payload libraryVerifyPayload) error {
store := writer
now := time.Now().UTC()
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
operationID: writer.attempt.OperationID}
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
if jobErr != nil {
// 作业行读不到就没有可登记的终态:直接失败,不落任何证据。
return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr)
}
if existing.State == governance.LibraryJobSucceeded {
// 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。
recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行")
return nil
}
if !config.ProductionReleaseRequestsEnabled {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行",
"capture")
}
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture",
"", "", "", "", now)
report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID,
func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) })
// 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。
evidence := governance.LibraryRestoreVerification{
ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID,
Scope: payload.Scope,
JobID: payload.LibraryJobID,
Status: governance.LibraryVerificationFailed,
StartedAt: now,
}
if report != nil {
evidence.StartedAt = report.StartedAt
evidence.SnapshotID = report.SnapshotID
evidence.Report = verificationReportObject(report)
}
if runErr == nil && report != nil && report.Status == "passed" {
evidence.Status = governance.LibraryVerificationPassed
evidence.FinishedAt = report.FinishedAt
} else {
evidence.FinishedAt = time.Now().UTC()
if report != nil && report.FinishedAt.After(evidence.StartedAt) {
evidence.FinishedAt = report.FinishedAt
}
switch {
case runErr != nil:
evidence.Failure = runErr.Error()
case report != nil && report.Status != "passed":
evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")"
default:
evidence.Failure = "恢复验证未产生报告"
}
}
if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil {
// 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize")
}
if runErr != nil || evidence.Status != governance.LibraryVerificationPassed {
held := report != nil && report.Status == "maintenance_held"
message := "恢复验证未通过:" + evidence.Failure
if held {
message += "(目标仍处于维护窗口,需要先按预备份恢复再重试)"
}
recovery := "未对目标执行任何写入,无需恢复"
if report != nil && report.SnapshotID != "" {
recovery = "目标已按预备份 " + report.SnapshotID + " 恢复并复核;下次验证会重新捕获"
}
snapshot := ""
if report != nil {
snapshot = report.SnapshotID
}
recorder.Stage(ctx, "finalize", "failed", message)
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
message, recovery, snapshot, snapshot, time.Now().UTC())
return errors.New(message)
}
finishedAt := evidence.FinishedAt
if finishedAt.IsZero() {
finishedAt = time.Now().UTC()
}
recorder.Stage(ctx, "finalize", "succeeded",
"恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize",
"", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt)
log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID)
return nil
}
// verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。
func verificationReportObject(report *libraryverify.Report) map[string]any {
if report == nil {
return nil
}
raw, err := json.Marshal(report)
if err != nil {
return map[string]any{"encode_error": err.Error()}
}
var object map[string]any
if err := json.Unmarshal(raw, &object); err != nil {
return map[string]any{"encode_error": err.Error()}
}
return object
}
// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。
//
// 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶
// 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED
// 仍作为显式覆盖保留(不删除旧部署能力),但页面不再需要它,也不再由页面写入。
func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface {
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
}, config appconfig.Config, scope string) (bool, string) {
if store != nil {
evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
switch {
case err != nil:
if config.ProductionBackupRestoreVerified {
return true, ""
}
return false, "读取恢复验证证据失败:" + err.Error()
case found:
if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok {
return true, ""
} else {
return false, reason
}
}
}
if config.ProductionBackupRestoreVerified {
return true, ""
}
return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)"
}
+53
View File
@@ -0,0 +1,53 @@
package main
import (
"context"
"strings"
"testing"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
)
// 同步闸口的"恢复可验证"判断必须以当前目标的证据为准:
// 没有证据 ⇒ 拒绝并说清怎么补;有 passed 证据 ⇒ 放行;只有失败证据 ⇒ 拒绝并带失败原因。
// 历史 env 覆盖只在证据缺失时兜底(页面不再写它)。
type verifyEvidenceStore struct {
evidence governance.LibraryRestoreVerification
found bool
}
func (s verifyEvidenceStore) LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) {
return s.evidence, s.found, nil
}
func TestLibraryRestoreEvidenceSatisfied(t *testing.T) {
now := time.Now().UTC()
passed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1", FinishedAt: now}}
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), passed, appconfig.Config{}, "internal"); !ok {
t.Fatalf("passed evidence must satisfy gate: %q", reason)
}
failed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-1",
Failure: "恢复后内容仍与预备份不一致", FinishedAt: now}}
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), failed, appconfig.Config{}, "internal"); ok ||
!strings.Contains(reason, "恢复后内容仍与预备份不一致") {
t.Fatalf("failed evidence must block with its own failure: ok=%v reason=%q", ok, reason)
}
missing := verifyEvidenceStore{}
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing, appconfig.Config{}, "internal"); ok ||
!strings.Contains(reason, "恢复验证") {
t.Fatalf("missing evidence must block and point at the remedy: ok=%v reason=%q", ok, reason)
}
// 历史部署的 env 覆盖:证据缺失时才生效,证据存在时不再起作用。
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing,
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); !ok {
t.Fatalf("legacy env override must still allow when no evidence: %q", reason)
}
if ok, _ := libraryRestoreEvidenceSatisfied(context.Background(), failed,
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); ok {
t.Fatal("evidence must win over the legacy env override")
}
}
+4
View File
@@ -110,6 +110,10 @@ const libraryRestoreJobType = "library.restore"
// libraryBaselineJobType 是"建立目标权威基线"的队列类型(与 API 侧常量同值)。
const libraryBaselineJobType = "library.baseline"
// libraryVerifyJobType 是"恢复验证"的队列类型:真实备份→canary→恢复→复核,
// 通过后才会产生同步闸口认的恢复证据。与 API 侧常量同值。
const libraryVerifyJobType = "library.verify"
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
+3 -2
View File
@@ -73,12 +73,13 @@ func (r LibraryWorkerRole) LibraryJobKinds() []string {
switch r {
case LibraryWorkerRoleReleaseExecutor:
return []string{string(governance.LibraryJobSync), string(governance.LibraryJobRestore),
string(governance.LibraryJobRecover), string(governance.LibraryJobBaseline)}
string(governance.LibraryJobRecover), string(governance.LibraryJobBaseline),
string(governance.LibraryJobVerify)}
case LibraryWorkerRoleCapture:
return []string{string(governance.LibraryJobSeal)}
default:
return []string{string(governance.LibraryJobSeal), string(governance.LibraryJobSync),
string(governance.LibraryJobRestore), string(governance.LibraryJobRecover),
string(governance.LibraryJobBaseline)}
string(governance.LibraryJobBaseline), string(governance.LibraryJobVerify)}
}
}
+3 -2
View File
@@ -63,13 +63,14 @@ func TestLoadRejectsUnknownLibraryWorkerRole(t *testing.T) {
// 每个角色收拾哪些 kind、谁拥有 DDL,都必须互斥且覆盖完整。
func TestLibraryWorkerRoleOwnershipMaps(t *testing.T) {
release := LibraryWorkerRoleReleaseExecutor
if got := release.LibraryJobKinds(); len(got) != 4 || got[0] != "sync" || got[1] != "restore" || got[2] != "recover" || got[3] != "baseline" {
if got := release.LibraryJobKinds(); len(got) != 5 || got[0] != "sync" || got[1] != "restore" ||
got[2] != "recover" || got[3] != "baseline" || got[4] != "verify" {
t.Fatalf("release-executor kinds = %v", got)
}
if got := LibraryWorkerRoleCapture.LibraryJobKinds(); len(got) != 1 || got[0] != "seal" {
t.Fatalf("capture kinds = %v", got)
}
if got := LibraryWorkerRoleFull.LibraryJobKinds(); len(got) != 5 {
if got := LibraryWorkerRoleFull.LibraryJobKinds(); len(got) != 6 {
t.Fatalf("full kinds = %v", got)
}
for _, role := range []LibraryWorkerRole{LibraryWorkerRoleFull, LibraryWorkerRoleCapture} {
+73 -2
View File
@@ -10,6 +10,8 @@ import (
"encoding/json"
"errors"
"fmt"
"net"
"os"
"os/exec"
"regexp"
"strconv"
@@ -127,12 +129,81 @@ func (r *Runner) runRemote(ctx context.Context, timeout time.Duration, remote st
if ctx.Err() != nil {
return text, fmt.Errorf("ssh %s 超时(%s)", r.config.Target, timeout)
}
// 主机侧脚本会用退出码表达"参数非法/有库失败",把输出带回去便于页面展示。
return text, fmt.Errorf("ssh %s 执行失败: %s", r.config.Target, firstLines(text, 6))
// 连接层失败与主机侧命令失败必须分开说:前者连脚本都没跑,
// 任何"脚本缺失/计划未下发"都是把未观测当成事实。
return text, hostFailure(r.config.Target, text)
}
return text, nil
}
// hostFailure 把 ssh 的非零退出分成两类:连不上,和连上了但远端命令失败。
func hostFailure(target, text string) error {
if connectionFailurePattern.MatchString(text) {
return fmt.Errorf("目标主机 %s 不可达(%s);未执行任何主机侧脚本,脚本与计划状态未知",
target, firstLines(text, 3))
}
// 主机侧脚本会用退出码表达"参数非法/有库失败",把输出带回去便于页面展示。
return fmt.Errorf("已连上 %s,但主机侧命令失败: %s", target, firstLines(text, 6))
}
// connectionFailurePattern 是 OpenSSH 客户端在"根本没建立连接"时打的原文。
// 只有命中它才允许说"不可达";其余非零退出属于远端命令自己的失败。
var connectionFailurePattern = regexp.MustCompile(`(?i)ssh: connect to host|Operation timed out|Connection timed out|Connection refused|Connection reset by peer|No route to host|Network is unreachable|Could not resolve hostname|Host key verification failed|Permission denied|Connection closed by`)
// Endpoint 读回挂载的 ssh 配置里本别名真实拨号的 host:port。
// 页面必须能说清"现在连的是哪个地址"——只显示别名时,改错文件也看不出来。
func (r *Runner) Endpoint() string {
if r == nil || r.config.SSHConfig == "" || r.config.Target == "" {
return ""
}
raw, err := os.ReadFile(r.config.SSHConfig)
if err != nil {
return ""
}
host, port := sshConfigEndpoint(string(raw), r.config.Target)
if host == "" {
return ""
}
if port == "" {
port = "22"
}
return net.JoinHostPort(host, port)
}
// sshConfigEndpoint 解析 `Host <alias>` 段的 HostName/Port(首个匹配段生效)。
// 只做"页面显示用"的最小解析,不替代 ssh 自己的配置语义。
func sshConfigEndpoint(config, alias string) (string, string) {
host, port := "", ""
active := false
for _, line := range strings.Split(config, "\n") {
fields := strings.Fields(strings.TrimSpace(line))
if len(fields) == 0 || strings.HasPrefix(fields[0], "#") {
continue
}
key := strings.ToLower(fields[0])
if key == "host" {
active = false
for _, pattern := range fields[1:] {
if pattern == alias {
active = true
break
}
}
continue
}
if !active || len(fields) < 2 {
continue
}
switch key {
case "hostname":
host = strings.Trim(fields[1], `"`)
case "port":
port = strings.Trim(fields[1], `"`)
}
}
return host, port
}
var scheduleAppliedPattern = regexp.MustCompile(
`SCHEDULE_APPLIED enabled=(\d)(?: time=(\S+))?(?: days=(\d+))?(?: months=(\d+))?(?: next=(\S+))?(?: cron=(\S+))?`)
+37
View File
@@ -3,6 +3,8 @@ package backuphost
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -188,6 +190,41 @@ func TestRunnerSurfacesHostFailure(t *testing.T) {
}
}
// 现场原错:ssh: connect to host 120.76.217.51 port 22: Operation timed out。
// 连不上时脚本根本没跑,只能说"不可达";断言配对的反面:不能说脚本缺失。
func TestHostFailureSeparatesUnreachableFromRemoteCommand(t *testing.T) {
unreachable := hostFailure("weknora-test", "ssh: connect to host 120.76.217.51 port 22: Operation timed out")
if !strings.Contains(unreachable.Error(), "不可达") || !strings.Contains(unreachable.Error(), "Operation timed out") {
t.Fatalf("连接超时应报不可达并带原错: %v", unreachable)
}
if strings.Contains(unreachable.Error(), "主机侧命令失败") || strings.Contains(unreachable.Error(), "脚本缺失") {
t.Fatalf("连接失败不能报成远端命令/脚本问题: %v", unreachable)
}
remote := hostFailure("weknora-test", "bash: /opt/weknora-test/logical-backup.sh: No such file or directory")
if !strings.Contains(remote.Error(), "已连上") || !strings.Contains(remote.Error(), "No such file") {
t.Fatalf("已连接但脚本失败时应原样带回原因: %v", remote)
}
if strings.Contains(remote.Error(), "不可达") {
t.Fatalf("已连上就不能说不可达: %v", remote)
}
}
// 页面要能说清"实际拨号的地址",否则改错 ssh_config 也看不出来。
func TestRunnerEndpointReadsDialedHostPort(t *testing.T) {
path := filepath.Join(t.TempDir(), "ssh_config")
config := "Host other\n HostName 10.0.0.9\n Port 2222\n\nHost weknora-test\n HostName 100.85.111.121\n HostKeyAlias 120.76.217.51\n"
if err := os.WriteFile(path, []byte(config), 0o600); err != nil {
t.Fatal(err)
}
runner := New(Config{SSHConfig: path, Target: "weknora-test", ScriptDir: "/opt/weknora-test"})
if got := runner.Endpoint(); got != "100.85.111.121:22" {
t.Fatalf("应报真实拨号地址(默认 22),得到 %q", got)
}
if got := New(Config{SSHConfig: filepath.Join(t.TempDir(), "missing"), Target: "weknora-test"}).Endpoint(); got != "" {
t.Fatalf("配置文件不可读时应返回空,不猜地址,得到 %q", got)
}
}
func TestRunnerRunNowReadsBackLastRun(t *testing.T) {
calls := 0
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
+10
View File
@@ -44,6 +44,10 @@ const (
// 读出来封存成快照并逐类校验,再登记成 production_effective。
// 它不写目标内容、不清空目标、不猜版本——基线只能来自真实捕获与校验。
LibraryJobBaseline LibraryJobKind = "baseline"
// LibraryJobVerify 是"恢复验证":在维护窗口内捕获目标 → 校验基线一致 →
// 写一个 canary 证明校验能发现差异 → 按预备份恢复 → 复核完全一致。
// 它是唯一能产生"生产可恢复"证据的作业,页面不再接受手填开关。
LibraryJobVerify LibraryJobKind = "verify"
)
type LibraryJobState string
@@ -173,6 +177,12 @@ func BaselineStages() []string {
return []string{"capture", "verify", "finalize"}
}
// VerifyStages 是恢复验证的阶段:捕获 → 基线校验 → 写 canary(证明校验有效)
// → 按预备份恢复 → 恢复后复核。canary 与 restore 都是真实写入,失败必须可见。
func VerifyStages() []string {
return []string{"capture", "verify", "canary", "restore", "finalize"}
}
// LibraryReleaseAuthorization 是生产发布授权:绑定目标、快照、参数摘要与有效期。
type LibraryReleaseAuthorization struct {
ID string `json:"id"`
+1 -1
View File
@@ -147,7 +147,7 @@ func ValidateLibraryOperation(op LibraryOperation) error {
return errors.New("library operation requires internal or external scope")
}
switch op.Kind {
case LibraryJobSeal, LibraryJobSync, LibraryJobRestore, LibraryJobBaseline, LibraryJobRecover:
case LibraryJobSeal, LibraryJobSync, LibraryJobRestore, LibraryJobBaseline, LibraryJobRecover, LibraryJobVerify:
default:
return fmt.Errorf("unsupported library operation kind %q", op.Kind)
}
@@ -0,0 +1,51 @@
package governance
import (
"strings"
"time"
)
// LibraryRestoreVerification 是一次**真实**恢复验证(备份 → canary → 恢复 → 复核)的登记。
//
// 为什么不能继续用环境变量 PRODUCTION_BACKUP_RESTORE_VERIFIED:那是一个可以手填的开关,
// 它既证明不了"有人真跑过恢复",也证明不了"跑的是当前目标"。这里的每条记录由 worker
// 在真实演练结束后写入,绑定 target_id(实例+库+索引+桶)与 scope,失败也照实留痕。
type LibraryRestoreVerification struct {
ID string `json:"id"`
Scope string `json:"scope"`
Status string `json:"status"` // passed | failed
SnapshotID string `json:"snapshot_id"`
JobID string `json:"job_id"`
Failure string `json:"failure,omitempty"`
Report map[string]any `json:"report,omitempty"`
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at"`
}
const (
LibraryVerificationPassed = "passed"
LibraryVerificationFailed = "failed"
)
// LibraryVerificationSatisfied 只回答一件事:这条证据能不能支撑"当前目标的恢复验证已通过"。
// 通过的条件必须是全部成立——状态 passed、scope 相同、快照非空、完成时间有效。
// target_id 的匹配由查询层(按当前 target_id 过滤)保证,这里不重复比较。
func LibraryVerificationSatisfied(verification LibraryRestoreVerification, scope string) (bool, string) {
if verification.Status != LibraryVerificationPassed {
reason := "最近一次恢复验证没有通过"
if detail := strings.TrimSpace(verification.Failure); detail != "" {
reason += ":" + detail
}
return false, reason
}
if verification.Scope != scope {
return false, "恢复验证证据属于另一个 scope(" + verification.Scope + ")"
}
if strings.TrimSpace(verification.SnapshotID) == "" {
return false, "恢复验证证据没有绑定快照,不能作为备份可恢复的证明"
}
if verification.FinishedAt.IsZero() {
return false, "恢复验证证据没有完成时间"
}
return true, ""
}
@@ -0,0 +1,59 @@
package governance
import (
"testing"
"time"
)
// 证据能不能支撑"当前目标可恢复"只由这四个条件回答:
// 状态 passed、scope 相同、绑定了快照、有完成时间。少任何一条都不算。
func TestLibraryVerificationSatisfied(t *testing.T) {
now := time.Now().UTC()
cases := []struct {
name string
verification LibraryRestoreVerification
scope string
want bool
reason string
}{
{name: "passed with snapshot", scope: "internal", want: true,
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal",
SnapshotID: "snap-1", FinishedAt: now}},
{name: "failed carries failure", scope: "internal", want: false, reason: "canary 后校验仍通过",
verification: LibraryRestoreVerification{Status: LibraryVerificationFailed, Scope: "internal",
SnapshotID: "snap-1", Failure: "canary 后校验仍通过", FinishedAt: now}},
{name: "wrong scope", scope: "internal", want: false, reason: "另一个 scope",
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "external",
SnapshotID: "snap-1", FinishedAt: now}},
{name: "no snapshot", scope: "internal", want: false, reason: "没有绑定快照",
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal",
FinishedAt: now}},
{name: "no finish time", scope: "internal", want: false, reason: "没有完成时间",
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal",
SnapshotID: "snap-1"}},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
ok, reason := LibraryVerificationSatisfied(testCase.verification, testCase.scope)
if ok != testCase.want {
t.Fatalf("satisfied=%v want=%v reason=%q", ok, testCase.want, reason)
}
if !testCase.want && testCase.reason != "" && !containsText(reason, testCase.reason) {
t.Fatalf("reason=%q want contains %q", reason, testCase.reason)
}
})
}
}
func containsText(haystack, needle string) bool {
return len(haystack) >= len(needle) && (haystack == needle || indexOf(haystack, needle) >= 0)
}
func indexOf(haystack, needle string) int {
for i := 0; i+len(needle) <= len(haystack); i++ {
if haystack[i:i+len(needle)] == needle {
return i
}
}
return -1
}
+12 -2
View File
@@ -24,6 +24,9 @@ type BackupScheduleStore interface {
type BackupHost interface {
Available() bool
Target() string
// Endpoint 是挂载的 ssh 配置里该别名真实拨号的 host:port;页面据此证明
// "连的是哪条通道",而不是只显示一个别名。
Endpoint() string
Show(context.Context) (backuphost.ShowResult, error)
Apply(context.Context, bool, string, int, int) (backuphost.ApplyResult, error)
RunNow(context.Context) (backuphost.RunResult, error)
@@ -82,8 +85,11 @@ type backupScheduleView struct {
// HostConfigured/HostReachable 区分"没接线"与"接线了但连不上"。
HostConfigured bool `json:"host_configured"`
HostTarget string `json:"host_target"`
HostReachable bool `json:"host_reachable"`
HostError string `json:"host_error"`
// HostEndpoint 是这条别名真实拨号的 host:port(来自挂载的 ssh 配置)。
// 连接失败时它就是"到底往哪连"的唯一证据,不能只给一个别名让人猜。
HostEndpoint string `json:"host_endpoint"`
HostReachable bool `json:"host_reachable"`
HostError string `json:"host_error"`
HostScheduleEnabled bool `json:"host_schedule_enabled"`
HostScheduleTime string `json:"host_schedule_time"`
@@ -281,6 +287,7 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
}
view.HostConfigured = config.BackupHost.Available()
view.HostTarget = config.BackupHost.Target()
view.HostEndpoint = config.BackupHost.Endpoint()
if !view.HostConfigured {
view.InSync = false
view.HostError = "目标主机凭据未挂载(BACKUP_HOST_SSH_CONFIG / BACKUP_HOST_TARGET)"
@@ -293,6 +300,9 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
if err != nil {
view.HostReachable = false
view.HostError = err.Error()
if view.HostEndpoint != "" {
view.HostError += "(拨号地址 " + view.HostEndpoint + ")"
}
view.InSync = false
} else {
view.HostReachable = true
+46
View File
@@ -46,6 +46,7 @@ func (store *fakeBackupScheduleStore) MarkBackupScheduleApplied(_ context.Contex
type fakeBackupHost struct {
available bool
target string
endpoint string
show backuphost.ShowResult
showErr error
apply backuphost.ApplyResult
@@ -59,6 +60,9 @@ type fakeBackupHost struct {
func (host *fakeBackupHost) Available() bool { return host.available }
func (host *fakeBackupHost) Target() string { return host.target }
func (host *fakeBackupHost) Endpoint() string {
return host.endpoint
}
func (host *fakeBackupHost) Show(context.Context) (backuphost.ShowResult, error) {
return host.show, host.showErr
}
@@ -166,6 +170,48 @@ func TestHandleBackupScheduleHostFlagsDrift(t *testing.T) {
}
}
func TestHandleBackupScheduleHostUnreachableSeparatesConnectionFromHostState(t *testing.T) {
// 现场原错:ssh connect to host 120.76.217.51 port 22 Operation timed out。
// 连接没建立 ⇒ 既不能报"脚本缺失",也不能报"计划未下发",只能报不可达 + 拨号地址。
host := &fakeBackupHost{
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
showErr: errors.New("目标主机 weknora-test 不可达(ssh: connect to host 120.76.217.51 port 22: Operation timed out);未执行任何主机侧脚本,脚本与计划状态未知"),
}
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
recorder := httptest.NewRecorder()
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
data := decodeData(t, recorder)
if data["host_reachable"] != false || data["in_sync"] != false {
t.Fatalf("连不上时不能报告可达或已同步: %+v", data)
}
message := data["host_error"].(string)
if !strings.Contains(message, "不可达") || !strings.Contains(message, "100.85.111.121:22") {
t.Fatalf("应报不可达并给出真实拨号地址: %q", message)
}
if strings.Contains(message, "脚本缺失") || strings.Contains(message, "未下发") {
t.Fatalf("未观测到的脚本/计划状态不能当成结论: %q", message)
}
}
func TestHandleBackupScheduleHostReportsMissingScriptOnlyAfterConnecting(t *testing.T) {
// 真正的脚本缺失:ssh 已连上,主机回的是脚本不存在 ⇒ 这时才允许说脚本缺失。
host := &fakeBackupHost{
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
showErr: errors.New("已连上 weknora-test,但主机侧命令失败: bash: /opt/weknora-test/install_backup_schedule.sh: No such file or directory"),
}
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
recorder := httptest.NewRecorder()
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
data := decodeData(t, recorder)
message := data["host_error"].(string)
if !strings.Contains(message, "已连上") || !strings.Contains(message, "No such file") {
t.Fatalf("已连接但脚本缺失时应原样带回主机原因: %q", message)
}
if strings.Contains(message, "不可达") {
t.Fatalf("已连上就不能说不可达: %q", message)
}
}
func TestHandleBackupScheduleHostUnconfiguredDegrades(t *testing.T) {
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, &fakeBackupHost{available: false})
recorder := httptest.NewRecorder()
+4 -5
View File
@@ -642,9 +642,7 @@ func librarySyncGateWithGrant(config Config, production libraryKnownView, prebac
if !config.ProductionReleaseRequestsEnabled {
gate.Reasons = append(gate.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
}
if !config.ProductionBackupRestoreVerified {
gate.Reasons = append(gate.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
}
gate.Reasons = append(gate.Reasons, restoreGateReasons(config)...)
// 生产端点以"整库目标是否配置"为准:整库同步直接写 PG/ES/对象,
// 不依赖 WeKnora 的 HTTP 端,所以不能拿它是否配置当门禁。
if strings.TrimSpace(config.LibraryProductionDatabaseURL) == "" {
@@ -814,7 +812,8 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi
grant, hasGrant := activeGrantFor(r.Context(), config, scope, target)
overview.Authorization = authorizationViewOf(grant, hasGrant)
overview.Gate = librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, target, hasGrant)
// 闸口按当前目标的真实恢复证据判断(没有证据时也接受历史 env 覆盖)。
overview.Gate = librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), scope), overview.Production, overview.Prebackup, target, hasGrant)
writeJSON(w, http.StatusOK, overview)
}
@@ -963,7 +962,7 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
return
}
}
gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant)
gate := librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), request.Scope), overview.Production, overview.Prebackup, request.Target, hasGrant)
if !gate.SyncAllowed {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
"code": "library_sync_blocked", "reasons": gate.Reasons,
+3 -9
View File
@@ -71,9 +71,7 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), request.Scope))...)
authorizationOK := false
if storeWithAuth, ok := config.Store.(interface {
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
@@ -169,9 +167,7 @@ func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config C
if !config.ProductionReleaseRequestsEnabled {
view.Reasons = append(view.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
}
if !config.ProductionBackupRestoreVerified {
view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
}
view.Reasons = append(view.Reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), scope))...)
digest, err := libraryRestoreParamsDigest(scope, snapshotID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
@@ -228,9 +224,7 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), source.Scope))...)
var digest string
switch source.Kind {
case governance.LibraryJobSync:
+219
View File
@@ -0,0 +1,219 @@
package httpapi
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"time"
"knowledge-governance/internal/governance"
)
// 恢复验证:页面点一次"跑恢复验证",后端建一个持久作业,worker 用与 CLI 同一份实现
// (internal/libraryverify)真实执行:捕获目标 → 基线校验 → 写 canary 证明校验有效 →
// 按预备份恢复 → 恢复后复核。通过与否只由这条证据表回答,页面不接受手填开关。
const libraryVerifyJobType = "library.verify"
type libraryVerifyRequest struct {
Scope string `json:"scope"`
}
// libraryVerificationView 是页面需要的全部事实:当前目标是谁、最近一次验证结果、
// 证据是否对当前目标成立、以及同步闸口能不能据此放行。
type libraryVerificationView struct {
Scope string `json:"scope"`
Target libraryVerificationTarget `json:"target"`
Verified bool `json:"verified"`
Reason string `json:"reason"`
Latest *governance.LibraryRestoreVerification `json:"latest,omitempty"`
History []governance.LibraryRestoreVerification `json:"history"`
Running bool `json:"running"`
JobID string `json:"job_id,omitempty"`
}
type libraryVerificationTarget struct {
Instance string `json:"instance"`
Project string `json:"project"`
InternalKB string `json:"internal_kb"`
ExternalKB string `json:"external_kb"`
Index string `json:"index"`
S3Bucket string `json:"s3_bucket"`
TenantID uint64 `json:"tenant_id"`
DatabaseSet bool `json:"database_configured"`
}
// libraryVerifyEnqueue 投递一次恢复验证作业(管理员)。
//
// 它不签发发布授权、不改版本:验证只回答"当前目标的预备份能不能恢复"。
// 目标是部署配置,不接受请求体指定——否则一次点击就能验证别的库并把证据挂到当前目标上。
func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
if r.Method != http.MethodPost {
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
return
}
identity, ok := sessionFrom(r.Context())
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
writeJSON(w, http.StatusForbidden, map[string]any{"error": "恢复验证仅管理员可发起"})
return
}
var request libraryVerifyRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
return
}
if request.Scope != "internal" && request.Scope != "external" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
return
}
store, ok := libraryStore(config)
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_verify_unavailable",
"error": "恢复验证存储不可用"})
return
}
// 验证要停写入者并在目标上写 canary:维护中或闸口关闭时直接拒绝,
// 不让页面投一个 worker 必然失败的作业。
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
writeJSON(w, http.StatusConflict, body)
return
}
if !config.ProductionReleaseRequestsEnabled {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
"code": "library_verify_blocked",
"reasons": []string{
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary",
},
})
return
}
digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobVerify,
map[string]any{"scope": request.Scope})
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"})
return
}
now := time.Now().UTC()
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
Kind: governance.LibraryJobVerify,
IdempotencyKey: "library.verify:" + request.Scope + ":" + now.Format("20060102T150405.000000000"),
RequestDigest: digest, Actor: identity.Username,
}, governance.VerifyStages(), false, now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active",
"error": "该目标已有在飞的写作业(同步/恢复/基线/验证互斥),等它结束再发起"})
return
}
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
"error": "同一幂等键对应了不同的请求"})
return
}
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": reused})
}
// libraryVerifications 读当前目标的恢复验证证据(页面轮询用)。
func libraryVerifications(w http.ResponseWriter, r *http.Request, config Config) {
scope, err := scopeFromQuery(r)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
return
}
view := libraryVerificationView{
Scope: scope,
History: []governance.LibraryRestoreVerification{},
Target: libraryVerificationTarget{
Instance: config.LibraryTargetInstance,
Project: config.LibraryTargetProject,
InternalKB: config.LibraryProductionInternalKB,
ExternalKB: config.LibraryProductionExternalKB,
Index: config.LibraryProductionIndex,
S3Bucket: config.LibraryProductionS3Bucket,
TenantID: config.LibraryProductionTenantID,
DatabaseSet: config.LibraryProductionDatabaseURL != "",
},
}
store, ok := config.Store.(interface {
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
LibraryRestoreVerifications(context.Context, string, int) ([]governance.LibraryRestoreVerification, error)
})
if !ok {
view.Reason = "当前部署没有恢复验证存储:无法读取证据"
writeJSON(w, http.StatusServiceUnavailable, view)
return
}
latest, found, err := store.LatestLibraryRestoreVerification(r.Context(), scope)
if err != nil {
view.Reason = "读取恢复验证证据失败:" + err.Error()
writeJSON(w, http.StatusServiceUnavailable, view)
return
}
if found {
view.Latest = &latest
view.JobID = latest.JobID
}
if history, err := store.LibraryRestoreVerifications(r.Context(), scope, 10); err == nil {
view.History = history
}
view.Verified, view.Reason = governance.LibraryVerificationSatisfied(latest, scope)
if !found {
view.Verified = false
view.Reason = "还没有对当前目标做过恢复验证:点「跑一次恢复验证」(真实备份 → canary → 恢复 → 复核)"
}
writeJSON(w, http.StatusOK, view)
}
// withRestoreEvidence 把"当前目标是否有真实恢复证据"注入 Config 的副本,
// 让同步/恢复闸口在同一个请求里按证据判断,而不是读一个可能被手填的 env。
// 读不到证据时不改写任何东西:闸口照旧因为缺证据而拒绝。
func (c Config) withRestoreEvidence(ctx context.Context, scope string) Config {
if ok, err := c.restoreEvidence(ctx, scope); err == nil {
c.restoreVerified = ok
c.restoreEvidenceReason = ""
} else {
c.restoreVerified = false
c.restoreEvidenceReason = err.Error()
}
return c
}
// restoreEvidence 读当前目标的最近一条证据并给出它能不能支撑"可恢复"。
func (c Config) restoreEvidence(ctx context.Context, scope string) (bool, error) {
store, ok := c.Store.(interface {
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
})
if !ok {
return false, errors.New("没有已核验的生产恢复演练证据")
}
latest, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
if err != nil {
return false, fmt.Errorf("读取恢复验证证据失败:%w", err)
}
if !found {
return false, errors.New("当前目标还没有恢复验证证据:先在「生产保护与恢复」里跑一次恢复验证")
}
ok, reason := governance.LibraryVerificationSatisfied(latest, scope)
if !ok {
return false, errors.New(reason)
}
return true, nil
}
// restoreGateReasons 是闸口对"恢复可验证"的统一判断:证据优先,
// 历史 env 覆盖仅在证据缺失时兜底(页面不再写入它)。
func restoreGateReasons(config Config) []string {
if config.restoreVerified || config.ProductionBackupRestoreVerified {
return nil
}
reason := config.restoreEvidenceReason
if reason == "" {
reason = "没有已核验的生产恢复演练证据"
}
return []string{reason}
}
+193
View File
@@ -0,0 +1,193 @@
package httpapi
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"knowledge-governance/internal/governance"
)
// 恢复验证的页面契约:
// - 只有管理员能发起(它会停写入者并在目标上写 canary);
// - 闸口关闭 / 维护中时直接拒绝,不投一个 worker 必然失败的作业;
// - 证据只由真实执行写入:没有证据 ⇒ verified=false,过期/换目标 ⇒ 失效。
//
// 这里不模拟"真跑一次恢复"(那需要真实目标);执行体本身在 internal/libraryverify,
// 由库内 CLI 与 worker 共用,属于隔离 fixture 之外的集成范围。
type verificationStore struct {
fakeLibraryStore
verifications []governance.LibraryRestoreVerification
}
func (s *verificationStore) LatestLibraryRestoreVerification(_ context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
for _, item := range s.verifications {
if item.Scope == scope {
return item, true, nil
}
}
return governance.LibraryRestoreVerification{}, false, nil
}
func (s *verificationStore) LibraryRestoreVerifications(_ context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
items := make([]governance.LibraryRestoreVerification, 0, len(s.verifications))
for _, item := range s.verifications {
if item.Scope == scope {
items = append(items, item)
}
}
if limit > 0 && len(items) > limit {
items = items[:limit]
}
return items, nil
}
func verifyRequest(body, role, username string) *http.Request {
request := httptest.NewRequest(http.MethodPost, "/api/v1/library/verifications", strings.NewReader(body))
request.Header.Set("Content-Type", "application/json")
ctx := context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: username, Role: role})
return request.WithContext(ctx)
}
func TestLibraryVerifyRequiresAdmin(t *testing.T) {
store := &verificationStore{}
recorder := httptest.NewRecorder()
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleEngineer, "eng"),
Config{Store: store, ProductionReleaseRequestsEnabled: true})
if recorder.Code != http.StatusForbidden {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
if store.createN != 0 {
t.Fatal("non-admin must not create a verify job")
}
}
func TestLibraryVerifyRefusesWhenWriteGateClosed(t *testing.T) {
store := &verificationStore{}
recorder := httptest.NewRecorder()
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
Config{Store: store, ProductionReleaseRequestsEnabled: false})
if recorder.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
body := decodeBody(t, recorder)
if body["code"] != "library_verify_blocked" {
t.Fatalf("body=%v", body)
}
if store.createN != 0 {
t.Fatal("closed gate must not create a verify job")
}
}
func TestLibraryVerifyEnqueuesVerifyJobWithCanaryStages(t *testing.T) {
store := &verificationStore{}
store.enqueueEchoRequestedID = true
recorder := httptest.NewRecorder()
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
Config{Store: store, ProductionReleaseRequestsEnabled: true})
if recorder.Code != http.StatusAccepted {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
if store.lastJob.Kind != governance.LibraryJobVerify || store.lastJob.Scope != "internal" {
t.Fatalf("job=%#v", store.lastJob)
}
if !strings.HasPrefix(store.lastJob.ID, "libverify-internal-") {
t.Fatalf("job id=%q", store.lastJob.ID)
}
// 阶段必须包含 canary:没有"写 canary 证明校验有效"这一步,验证就只是自证。
want := governance.VerifyStages()
if len(want) != 5 || want[2] != "canary" || want[3] != "restore" || want[4] != "finalize" {
t.Fatalf("verify stages=%v", want)
}
}
func TestLibraryVerificationsWithoutEvidenceIsNotVerified(t *testing.T) {
store := &verificationStore{}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
libraryVerifications(recorder, request, Config{Store: store})
if recorder.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
body := decodeBody(t, recorder)
if body["verified"] != false {
t.Fatalf("verified must be false without evidence: %v", body)
}
if reason, _ := body["reason"].(string); !strings.Contains(reason, "还没有对当前目标做过恢复验证") {
t.Fatalf("reason=%q", reason)
}
}
func TestLibraryVerificationsWithPassedEvidenceIsVerified(t *testing.T) {
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
ID: "libverify-internal-job1", Scope: "internal", Status: governance.LibraryVerificationPassed,
SnapshotID: "snap-1", JobID: "job1", StartedAt: time.Now().UTC().Add(-time.Minute),
FinishedAt: time.Now().UTC(),
}}}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
libraryVerifications(recorder, request, Config{Store: store})
body := decodeBody(t, recorder)
if body["verified"] != true {
t.Fatalf("verified must be true with passed evidence: %v", body)
}
}
func TestLibraryVerificationsFailedEvidenceIsNotVerified(t *testing.T) {
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
ID: "libverify-internal-job2", Scope: "internal", Status: governance.LibraryVerificationFailed,
SnapshotID: "snap-2", JobID: "job2", Failure: "恢复后内容仍与预备份不一致",
StartedAt: time.Now().UTC().Add(-time.Minute), FinishedAt: time.Now().UTC(),
}}}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
libraryVerifications(recorder, request, Config{Store: store})
body := decodeBody(t, recorder)
if body["verified"] != false {
t.Fatalf("failed evidence must not verify: %v", body)
}
if reason, _ := body["reason"].(string); !strings.Contains(reason, "恢复后内容仍与预备份不一致") {
t.Fatalf("reason=%q", reason)
}
}
// 同步闸口必须消费同一 target 的证据:有 passed 证据即放行,
// 只有失败证据时仍然拒绝并给出失败原因(不是笼统"没有证据")。
func TestLibrarySyncGateConsumesRestoreEvidence(t *testing.T) {
base := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40}
base.Store = &verificationStore{verifications: []governance.LibraryRestoreVerification{{
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1",
JobID: "job1", FinishedAt: time.Now().UTC(),
}}}
allowed := librarySyncGateWithGrant(base.withRestoreEvidence(context.Background(), "internal"),
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
if !allowed.SyncAllowed {
t.Fatalf("gate should allow with passed evidence: %v", allowed.Reasons)
}
noEvidence := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
Store: &verificationStore{}}.withRestoreEvidence(context.Background(), "internal")
blocked := librarySyncGateWithGrant(noEvidence,
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
if blocked.SyncAllowed {
t.Fatalf("gate without env and without evidence must block: %v", blocked.Reasons)
}
if len(blocked.Reasons) == 0 || !strings.Contains(strings.Join(blocked.Reasons, ";"), "恢复验证") {
t.Fatalf("blocked reasons=%v", blocked.Reasons)
}
failedConfig := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
Store: &verificationStore{verifications: []governance.LibraryRestoreVerification{{
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-2",
JobID: "job2", Failure: "canary 后校验仍通过", FinishedAt: time.Now().UTC(),
}}}}
reason := strings.Join(restoreGateReasons(failedConfig.withRestoreEvidence(context.Background(), "internal")), ";")
if !strings.Contains(reason, "canary 后校验仍通过") {
t.Fatalf("失败证据的失败原因必须出现在闸口:%q", reason)
}
}
+54
View File
@@ -6,6 +6,7 @@ import (
"strings"
"time"
"knowledge-governance/internal/pgdsn"
"knowledge-governance/internal/platform/weknora"
)
@@ -33,6 +34,55 @@ type productionBindingView struct {
ProductionBaseURL string `json:"production_base_url"`
ProductionConnected bool `json:"production_connected"`
ProductionMessage string `json:"production_message"`
// LegacyScope 说明这条记录属于哪条发布路径。整库发布(封存/同步/恢复)不读它,
// 只读下面的 WholeTarget;页面必须把两者分开写,否则会把旧文档级目标当成整库目标。
LegacyScope string `json:"legacy_scope"`
// WholeTarget 是整库发布真正写入的目标(来自部署环境 LIBRARY_PRODUCTION_*),
// 与上面可编辑的文档级绑定是两条独立路径。
WholeTarget libraryPublishTargetView `json:"whole_target"`
}
// libraryPublishTargetView 是"整库发布到底写哪儿"的只读投影。
// 为什么由后端给出:页面上的 kb 名称/文档数来自生产端列表接口,而整库写入用的是
// 另一组环境配置;让页面自己拼会把两条路径混成一个。
type libraryPublishTargetView struct {
Configured bool `json:"configured"`
Source string `json:"source"`
TenantID uint64 `json:"tenant_id"`
InternalKBID string `json:"internal_kb_id"`
ExternalKBID string `json:"external_kb_id"`
Database string `json:"database"`
ElasticIndex string `json:"elastic_index"`
ObjectBucket string `json:"object_bucket"`
MissingReason string `json:"missing_reason"`
}
// libraryPublishTarget 从 config 派生整库目标;缺项照实说缺哪一项,不猜默认值。
func (config Config) libraryPublishTarget() libraryPublishTargetView {
view := libraryPublishTargetView{
Source: "部署环境 LIBRARY_PRODUCTION_*",
TenantID: config.LibraryProductionTenantID,
InternalKBID: config.LibraryProductionInternalKB,
ExternalKBID: config.LibraryProductionExternalKB,
Database: pgdsn.Endpoint(config.LibraryProductionDatabaseURL),
ElasticIndex: config.LibraryProductionIndex,
ObjectBucket: config.LibraryProductionS3Bucket,
}
missing := make([]string, 0, 4)
if config.LibraryProductionDatabaseURL == "" {
missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL")
}
if view.InternalKBID == "" {
missing = append(missing, "LIBRARY_PRODUCTION_KB_INTERNAL")
}
if config.LibraryProductionIndex == "" {
missing = append(missing, "LIBRARY_PRODUCTION_INDEX")
}
view.Configured = len(missing) == 0
if len(missing) > 0 {
view.MissingReason = "未配置:" + strings.Join(missing, "、")
}
return view
}
// productionTargetBaseURL is the production endpoint the picker reads from: the
@@ -79,6 +129,10 @@ func (config Config) productionBindingPayload(ctx context.Context) (productionBi
InternalKBName: names[storedInternal], ExternalKBName: names[storedExternal],
UpdatedBy: updatedBy, ProductionBaseURL: config.productionTargetBaseURL(ctx),
ProductionConnected: message == "", ProductionMessage: message,
// 这张表服务的是"文档级发布"(单篇文档写生产)。整库发布读的是部署环境
// LIBRARY_PRODUCTION_*,两者不能互相顶替,所以页面必须分开显示。
LegacyScope: "文档级发布(单篇文档写生产)· 旧路径",
WholeTarget: config.libraryPublishTarget(),
}, nil
}
@@ -178,3 +178,50 @@ func TestProductionBindingOptionsReportUnavailableReason(t *testing.T) {
t.Fatalf("status = %d body %s", recorder.Code, recorder.Body.String())
}
}
// 页面必须说清"整库发布真正写哪个 KB/端点"。这条投影只能来自部署配置,
// 不能从可编辑的文档级绑定推出来,也不能把口令带进响应。
func TestLibraryPublishTargetIsDerivedFromDeploymentConfig(t *testing.T) {
config := Config{
// 只放形状,不放任何可用凭据:口令段是占位符,主机是 .invalid。
LibraryProductionDatabaseURL: "postgres://governance@db.invalid:5432/weknora_test?sslmode=disable",
LibraryProductionInternalKB: "a8e9c8ad-411c-4c53-8421-599d29e9b179",
LibraryProductionExternalKB: "2e980686-2233-4c4e-b5a6-8b0069477b8e",
LibraryProductionTenantID: 10000,
LibraryProductionIndex: "weknora_test_chunks",
LibraryProductionS3Bucket: "dobot-weknora-test",
}
target := config.libraryPublishTarget()
if !target.Configured || target.MissingReason != "" {
t.Fatalf("完整配置应判为已配置: %+v", target)
}
if target.InternalKBID != "a8e9c8ad-411c-4c53-8421-599d29e9b179" || target.TenantID != 10000 {
t.Fatalf("必须回显实际整库目标而不是占位: %+v", target)
}
if target.Database != "db.invalid:5432/weknora_test" {
t.Fatalf("端点应只含 host:port/db,得到 %q", target.Database)
}
// 回显只能是端点:连接串里的用户信息(口令、账号)一个字符都不能进响应。
if strings.Contains(target.Database, "@") || strings.Contains(target.Source, "@") ||
strings.Contains(target.Source, "postgres://") {
t.Fatalf("响应里不能出现连接串用户信息: %+v", target)
}
if target.ElasticIndex != "weknora_test_chunks" || target.ObjectBucket != "dobot-weknora-test" {
t.Fatalf("索引与桶也要显示: %+v", target)
}
}
func TestLibraryPublishTargetNamesMissingConfigInsteadOfGuessing(t *testing.T) {
target := Config{}.libraryPublishTarget()
if target.Configured {
t.Fatal("缺配置时不能报告已配置")
}
for _, key := range []string{"LIBRARY_PRODUCTION_DATABASE_URL", "LIBRARY_PRODUCTION_KB_INTERNAL"} {
if !strings.Contains(target.MissingReason, key) {
t.Fatalf("缺失说明应点名 %s,得到 %q", key, target.MissingReason)
}
}
if target.InternalKBID != "" || target.Database != "(未配置)" {
t.Fatalf("缺配置就不要编造目标: %+v", target)
}
}
+18 -2
View File
@@ -181,6 +181,9 @@ type Config struct {
LibraryProductionInternalKB string
LibraryProductionExternalKB string
LibraryProductionTenantID uint64
// 整库发布还落在这些后端上;页面要显示"到底写哪儿",只给 KB ID 不够。
LibraryProductionIndex string
LibraryProductionS3Bucket string
// LibraryScopeProbe 让测试注入"目标整库是否为空"的探测;留空即用真实连接。
// 它不是运行时开关:生产部署不设置它,那次探测仍然直连目标库逐表计数。
LibraryScopeProbe func(ctx context.Context, scope string) (bool, map[string]int64, error)
@@ -198,8 +201,13 @@ type Config struct {
RequireWeKnoraReady bool
ProductionReleaseRequestsEnabled bool
// TestAutoReviewPolicy 只用于展示:让运维在界面上看到"现在自动批准到什么程度"。
TestAutoReviewPolicy string
ProductionBackupRestoreVerified bool
TestAutoReviewPolicy string
ProductionBackupRestoreVerified bool
// restoreVerified / restoreEvidenceReason 由请求期的证据读取填入(withRestoreEvidence):
// 闸口按"当前目标是否有真实恢复验证证据"判断,而不是按可手填的 env。
// 它们不可由配置注入,避免把证据变成另一个开关。
restoreVerified bool
restoreEvidenceReason string
ProductionReleaseAuthorizerRef string
ProductionReleaseAuthorizationToken string
// DingtalkSources 枚举「我账号下能看到」的钉钉知识库与钉盘空间(只读,dws)。
@@ -556,6 +564,14 @@ func NewHandler(config Config) (http.Handler, error) {
mux.HandleFunc("POST /api/v1/library/baselines", func(w http.ResponseWriter, r *http.Request) {
libraryBaselineEnqueue(w, r, config)
})
// 恢复验证:真实备份 → canary → 恢复 → 复核,通过后证据按当前目标登记,
// 同步闸口据此放行(不再需要页面/环境变量手填 verified)。
mux.HandleFunc("POST /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
libraryVerifyEnqueue(w, r, config)
})
mux.HandleFunc("GET /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
libraryVerifications(w, r, config)
})
// 全部 main 提交时间线(读本库缓存,不直接打 Gitea)
mux.HandleFunc("GET /api/v1/version-history", func(w http.ResponseWriter, r *http.Request) {
versionHistoryList(w, r, config)
@@ -1,4 +1,4 @@
package main
package libraryverify
import (
"context"
+266
View File
@@ -0,0 +1,266 @@
// Package libraryverify 在目标实例上做一次**真实**的恢复验证(原 cmd/librarydrill 的实现):
//
// 维护隔离(关入口/停写入者/证明排空)
// → 捕获目标整库(真预备份,落归档)
// → 校验目标 == 预备份(基线一致)
// → 写入一个 canary(证明"校验确实能发现内容变化")
// → 按预备份恢复
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
//
// 为什么搬进 internal:页面要能点一次"恢复验证",worker 必须在进程内执行同一份实现,
// 不能再要求运维 SSH 上去跑 CLI。命令行使同一个 Run,不允许两份实现。
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境,不进命令行、不打印。
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
package libraryverify
import (
"context"
"crypto/rand"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarystack"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
type Verification struct {
SnapshotID string `json:"snapshot_id"`
Verified bool `json:"verified"`
Error string `json:"error,omitempty"`
At time.Time `json:"at"`
}
type Report struct {
Status string `json:"status"` // passed | failed | maintenance_held
Scope string `json:"scope"`
Target string `json:"target"`
SnapshotID string `json:"snapshot_id"`
Project string `json:"project"`
Counts map[string]int64 `json:"counts"`
BaselineCheck *Verification `json:"baseline_check"`
DriftCheck *Verification `json:"drift_check"` // 必须失败:证明确实能发现差异
RestoreCheck *Verification `json:"restore_check"`
Canary string `json:"canary"`
Failure string `json:"failure,omitempty"`
Stage string `json:"stage,omitempty"`
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at"`
Until map[string]string `json:"until,omitempty"`
}
// Progress 把演练的阶段推进交给调用方(worker 写 library_job_stages,CLI 打日志)。
type Progress func(stage, state, detail string)
func (p Progress) report(stage, state, detail string) {
if p != nil {
p(stage, state, detail)
}
}
// Run 执行一次演练。jobID 同时是维护窗口的持有者标识,必须与业务作业 id 一致,
// 否则崩溃恢复时窗口与作业对不上。
func Run(ctx context.Context, config appconfig.Config, scope, jobID string, progress Progress) (*Report, error) {
if scope != "internal" && scope != "external" {
return nil, errors.New("scope must be internal or external")
}
report := &Report{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
fail := func(stage string, cause error, held bool) (*Report, error) {
report.Stage = stage
report.Failure = cause.Error()
report.FinishedAt = time.Now().UTC()
if held {
report.Status = "maintenance_held"
}
progress.report(stage, "failed", cause.Error())
return report, cause
}
progress.report("capture", "running", "打开目标栈与维护管理器")
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
if err != nil {
return fail("capture", err, false)
}
defer target.Close()
manager, err := librarystack.OpenMaintenance(config)
if err != nil {
return fail("capture", err, false)
}
defer manager.Close()
if err := manager.VerifyTopology(ctx); err != nil {
return fail("capture", err, false)
}
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
if err != nil {
return fail("capture", err, false)
}
defer control.Close()
defer coordinator.Close()
var (
snapshotID string
counts map[string]int64
canary string
contentTouched bool
baseline Verification
drift Verification
restored Verification
)
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
start := time.Now()
current, err := target.Current(ctx)
if err != nil {
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
}
counts = current.Counts
report.Counts = counts
// 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。
backup, err := backups.Prepare(ctx, "", current)
if err != nil {
return "", errors.New("捕获目标整库失败:" + err.Error())
}
snapshotID = backup.SnapshotID
report.SnapshotID = snapshotID
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
return "", errors.New("记录预备份身份失败:" + err.Error())
}
progress.report("capture", "succeeded",
fmt.Sprintf("已捕获目标整库快照 %s(%d 字节,用时 %s)", snapshotID, backup.Bytes, time.Since(start).Round(time.Second)))
log.Printf("verify: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
// 2) 基线校验:预备份必须与目标完全一致。
progress.report("verify", "running", "逐表/索引/资源比对目标与预备份")
baseline = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
baseline.Error = err.Error()
report.BaselineCheck = &baseline
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
}
baseline.Verified = true
report.BaselineCheck = &baseline
progress.report("verify", "succeeded", "预备份与目标逐类一致")
// Canary 是隔离的标签行,空目标上也一样:绝不为了"让演练能跑"去种文档或改既有行。
var random [16]byte
if _, err := rand.Read(random[:]); err != nil {
return "", err
}
canary = fmt.Sprintf("%x", random[:])
identityValue := target.Identity()
var exists bool
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
return "", err
}
if exists {
return "", errors.New("canary identity collision; no write attempted")
}
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
report.Canary = canary
progress.report("canary", "running", "写入一个隔离 canary,验证校验能发现差异")
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
func(ctx context.Context) error {
if err := window.MarkContentRisk(ctx); err != nil {
return err
}
contentTouched = true
return nil
}, func(ctx context.Context) error {
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
return err
}); err != nil {
return "", fmt.Errorf("canary preparation/write failed: %w", err)
}
drift = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
report.DriftCheck = &drift
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
} else {
drift.Error = verifyErr.Error()
}
report.DriftCheck = &drift
progress.report("canary", "succeeded", "校验按预期发现了 canary 差异")
log.Printf("verify: drift detected as expected (%s)", drift.Error)
// 4) 按预备份恢复。恢复只删掉持久写集合里那个新 canary 身份。
progress.report("restore", "running", "按预备份恢复目标整库")
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
WriteSet: &ledger}); err != nil {
return "", errors.New("按预备份恢复失败:" + err.Error())
}
// 5) 恢复后逐类复核:必须与预备份完全一致。
restored = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
restored.Error = err.Error()
report.RestoreCheck = &restored
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
}
restored.Verified = true
report.RestoreCheck = &restored
progress.report("restore", "succeeded", "恢复后与预备份逐类一致")
log.Printf("verify: restored and verified snapshot=%s", snapshotID)
return snapshotID, nil
}
progress.report("finalize", "running", "关闭维护窗口并按原状态恢复目标")
window, windowErr := coordinator.RunWindow(ctx, jobID,
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
func(error) bool { return !contentTouched || restored.Verified })
report.Until = map[string]string{"job": jobID, "window_state": window.State}
if windowErr != nil || window.State != "succeeded" {
cause := windowErr
if cause == nil {
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
}
if window.State == "held" {
return fail("finalize", cause, true)
}
return fail("finalize", cause, false)
}
report.Status = "passed"
report.FinishedAt = time.Now().UTC()
progress.report("finalize", "succeeded", "维护窗口已关闭,目标恢复原运行状态")
return report, nil
}
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error {
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
if err := ledger.Seal(); err != nil {
return err
}
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
return err
}
if err := mark(ctx); err != nil {
return err
}
return write(ctx)
}
// RecoverWindows 把非终态的维护窗口按原运行状态恢复(服务起来、入口验证)。
// 演练失败但还没写目标内容时,用它把目标从"停住"状态救回来;治理记录由作业/接口那侧负责。
func RecoverWindows(config appconfig.Config) error {
manager, err := librarystack.OpenMaintenance(config)
if err != nil {
return err
}
defer manager.Close()
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
if err != nil {
return err
}
defer control.Close()
defer coordinator.Close()
return coordinator.Recover(context.Background())
}
@@ -33,7 +33,8 @@ func (s *Store) createLibraryDispatch(ctx context.Context, job governance.Librar
return created, true, nil
}
kind := "library." + string(created.Kind)
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore && created.Kind != governance.LibraryJobBaseline {
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore &&
created.Kind != governance.LibraryJobBaseline && created.Kind != governance.LibraryJobVerify {
return created, reused, errors.New("unsupported library dispatch kind")
}
queueID, kind, raw, err := governance.LibraryQueueRequest(handoff, created.ID, created.Scope, created.TargetSnapshotID, kind)
@@ -0,0 +1,98 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
// 恢复验证证据:只由 worker 在真实演练结束后写入;这里只做落库与按当前目标读取。
//
// 读取永远带 target_id = 当前部署目标:换库/换索引/换桶之后旧证据自动失效,
// 不需要任何人去"记得把开关关掉"。
// SaveLibraryRestoreVerification 幂等登记一次演练结果(同一 target+scope+job 只留一条)。
func (s *Store) SaveLibraryRestoreVerification(ctx context.Context, verification governance.LibraryRestoreVerification) error {
if verification.ID == "" || verification.JobID == "" {
return fmt.Errorf("restore verification requires id and job id")
}
if verification.Scope != "internal" && verification.Scope != "external" {
return fmt.Errorf("restore verification scope %q is invalid", verification.Scope)
}
if verification.Status != governance.LibraryVerificationPassed && verification.Status != governance.LibraryVerificationFailed {
return fmt.Errorf("restore verification status %q is invalid", verification.Status)
}
report, err := json.Marshal(verification.Report)
if err != nil {
return fmt.Errorf("encode restore verification report: %w", err)
}
_, err = s.db.Exec(ctx, `
INSERT INTO library_restore_verifications (id, target_id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
ON CONFLICT (target_id, scope, job_id) DO UPDATE
SET status = EXCLUDED.status, snapshot_id = EXCLUDED.snapshot_id, failure = EXCLUDED.failure,
report = EXCLUDED.report, started_at = EXCLUDED.started_at, finished_at = EXCLUDED.finished_at`,
verification.ID, s.libraryTargetID, verification.Scope, verification.Status,
verification.SnapshotID, verification.JobID, verification.Failure, report,
verification.StartedAt.UTC(), verification.FinishedAt.UTC())
if err != nil {
return fmt.Errorf("save restore verification: %w", err)
}
return nil
}
// LatestLibraryRestoreVerification 按完成时间取当前目标该 scope 的最近一条证据。
// 没有记录返回 (zero,false,nil):那不是错误,页面据此说"还没有验证过"。
func (s *Store) LatestLibraryRestoreVerification(ctx context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
var (
item governance.LibraryRestoreVerification
report []byte
)
err := s.db.QueryRow(ctx, `
SELECT id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at
FROM library_restore_verifications
WHERE target_id=$1 AND scope=$2
ORDER BY finished_at DESC, created_at DESC
LIMIT 1`, s.libraryTargetID, scope).Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID,
&item.JobID, &item.Failure, &report, &item.StartedAt, &item.FinishedAt)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return governance.LibraryRestoreVerification{}, false, nil
}
return governance.LibraryRestoreVerification{}, false, fmt.Errorf("load restore verification: %w", err)
}
item.Report = decodeEvidenceObject(report)
return item, true, nil
}
// LibraryRestoreVerifications 是页面用的轻量清单(最近 N 条),按目标过滤。
func (s *Store) LibraryRestoreVerifications(ctx context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
if limit <= 0 {
limit = 10
}
rows, err := s.db.Query(ctx, `
SELECT id, scope, status, snapshot_id, job_id, failure, started_at, finished_at
FROM library_restore_verifications
WHERE target_id=$1 AND scope=$2
ORDER BY finished_at DESC, created_at DESC
LIMIT $3`, s.libraryTargetID, scope, limit)
if err != nil {
return nil, fmt.Errorf("list restore verifications: %w", err)
}
defer rows.Close()
items := make([]governance.LibraryRestoreVerification, 0, limit)
for rows.Next() {
var item governance.LibraryRestoreVerification
if err := rows.Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID, &item.JobID,
&item.Failure, &item.StartedAt, &item.FinishedAt); err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
@@ -0,0 +1,13 @@
DROP INDEX IF EXISTS library_restore_verifications_latest;
DROP TABLE IF EXISTS library_restore_verifications;
-- 先把 verify 作业清掉(它没有历史数据,只可能来自本版本),再恢复旧的 kind 约束。
DELETE FROM library_jobs WHERE kind = 'verify';
ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check;
ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check
CHECK (kind IN ('seal','sync','restore','baseline'));
DROP INDEX IF EXISTS library_jobs_write_target;
CREATE UNIQUE INDEX library_jobs_write_target
ON library_jobs(scope)
WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline');
@@ -0,0 +1,37 @@
-- 恢复验证成为一种独立业务作业,并且它的证据只能由真实演练写入。
--
-- 为什么需要:闸门原来读 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个纯 env 开关,
-- 谁把它写成 true 都能"证明"生产可恢复;页面也没有入口去真跑一次恢复。
-- 现在 verified 只能来自 library_restore_verifications 里 passed 的真实记录。
--
-- verify 与 sync/restore/baseline 共用同一把写互斥:它同样要停写入者、开维护窗口、
-- 在目标上写 canary 再恢复,不能和别的写作业并发。
ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check;
ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check
CHECK (kind IN ('seal','sync','restore','baseline','verify'));
DROP INDEX IF EXISTS library_jobs_write_target;
CREATE UNIQUE INDEX library_jobs_write_target
ON library_jobs(scope)
WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline','verify');
-- target_id 是 appconfig.LibraryTargetIdentity():换目标(库/索引/桶/实例)后旧证据不再适用,
-- 这是"证据必须对得上当前目标"的落库形态。
CREATE TABLE IF NOT EXISTS library_restore_verifications (
id text PRIMARY KEY,
target_id text NOT NULL CHECK (target_id <> ''),
scope text NOT NULL CHECK (scope IN ('internal','external')),
status text NOT NULL CHECK (status IN ('passed','failed')),
snapshot_id text NOT NULL DEFAULT '',
job_id text NOT NULL CHECK (job_id <> ''),
failure text NOT NULL DEFAULT '',
report jsonb,
started_at timestamptz NOT NULL,
finished_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
-- 同一作业只留一条;重复上报是幂等重放,不是新证据。
CONSTRAINT library_restore_verifications_job UNIQUE (target_id, scope, job_id)
);
CREATE INDEX IF NOT EXISTS library_restore_verifications_latest
ON library_restore_verifications (target_id, scope, status, finished_at DESC);
@@ -11,8 +11,15 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 78 {
t.Fatalf("migration count = %d, want 78", len(migrations))
if len(migrations) != 79 {
t.Fatalf("migration count = %d, want 79", len(migrations))
}
// 79 是恢复验证:verify 成为独立业务作业并与写作业共用互斥,
// 证据按 target_id 落库(换目标旧证据自动失效)。
if verification := migrations[78]; verification.Version != 79 ||
!strings.Contains(verification.UpSQL, "library_restore_verifications") ||
!strings.Contains(verification.UpSQL, "'verify'") {
t.Fatalf("migration 79 = %#v", verification)
}
// 78 是 DS01 恢复标记派发台账:唯一 key、单调状态与激活登记;只动治理库。
if dispatch := migrations[77]; dispatch.Version != 78 ||
+26
View File
@@ -22,6 +22,7 @@ const base: BackupSchedule = {
last_applied_summary: '每天 03:30 · 保留 7 天 · 每月归档 3 份',
host_configured: true,
host_target: 'weknora-test',
host_endpoint: '100.85.111.121:22',
host_reachable: true,
host_error: '',
host_schedule_enabled: true,
@@ -88,6 +89,9 @@ describe('BackupScheduleView', () => {
'重新下发当前设置',
'立即备份一次',
'weknora-test · 已连接',
'拨号地址',
'100.85.111.121:22',
'全库,不是 scope 快照',
'每天 03:30 · 保留 7 天 · 每月 3 份',
'已一致',
'20261009T052909Z',
@@ -120,6 +124,28 @@ describe('BackupScheduleView', () => {
expect(html).toContain('disabled')
})
it('连不上主机:报连接失败与拨号地址,脚本状态写未知而不是"缺失"', () => {
const html = view({
host_reachable: false,
host_endpoint: '120.76.217.51:22',
host_error:
'目标主机 weknora-test 不可达(ssh: connect to host 120.76.217.51 port 22: Operation timed out);未执行任何主机侧脚本,脚本与计划状态未知(拨号地址 120.76.217.51:22)',
in_sync: false,
host_script_present: false,
})
expect(html).toContain('weknora-test · 连接失败')
expect(html).toContain('120.76.217.51:22')
expect(html).toContain('未知(未连上主机,未执行主机侧脚本)')
expect(html).not.toContain('缺失')
})
it('已连上且脚本真的不在:这时才显示"缺失"', () => {
const html = view({ host_reachable: true, host_script_present: false, in_sync: false })
expect(html).toContain('weknora-test · 已连接')
expect(html).toContain('缺失')
expect(html).not.toContain('未知(未连上主机')
})
it('待下发(主机上的计划与设置不一致)时提示', () => {
const html = view({ host_schedule_time: '05:00', in_sync: false })
expect(html).toContain('待下发')
+16 -5
View File
@@ -91,6 +91,12 @@ export function BackupScheduleView(props: BackupScheduleViewProps) {
: schedule.host_reachable
? `${schedule.host_target} · 已连接`
: `${schedule.host_target} · 连接失败`
// 只有真的连上并读回主机状态时,才能说脚本在不在;连不上是"未知"。
const scriptLine = !schedule.host_reachable
? '未知(未连上主机,未执行主机侧脚本)'
: schedule.host_script_present
? '存在'
: '缺失'
return (
<section className="settings-panel wide storage-settings-panel" id="settings-storage">
@@ -115,11 +121,12 @@ export function BackupScheduleView(props: BackupScheduleViewProps) {
<dl>
<dt>目标主机</dt>
<dd>{hostLine}</dd>
<dt>拨号地址</dt>
<dd>{schedule.host_endpoint || '—'}</dd>
<dt>备份脚本</dt>
<dd>{scriptLine}</dd>
<dt>主机 cron</dt>
<dd>
{schedule.host_cron_service ? `服务 ${schedule.host_cron_service}` : '—'}
{schedule.host_script_present ? '' : ' · 备份脚本缺失'}
</dd>
<dd>{schedule.host_cron_service ? `服务 ${schedule.host_cron_service}` : '—'}</dd>
<dt>最近一次备份</dt>
<dd>
{schedule.last_run_stamp || '—'}
@@ -213,8 +220,12 @@ export function BackupScheduleView(props: BackupScheduleViewProps) {
{busy === 'apply' ? '正在下发…' : '重新下发当前设置'}
</button>
<button onClick={onRunNow} disabled={busy !== null || !schedule.host_configured || schedule.running}>
{schedule.running ? '备份进行中…' : '立即备份一次'}
{schedule.running ? '备份进行中…' : '立即备份一次(全库,不是 scope 快照)'}
</button>
<p className="hint">
「立即备份一次」执行的是主机上的整库逻辑备份(该主机上全部业务库的 pg_dump 目录),
<strong>不是</strong>本次对内/对外 scope 的整库快照;scope 快照在「发布管理 → 整库版本」里封存。
</p>
{message ? (
<p className="success-message" role="status">
{message}
+101
View File
@@ -0,0 +1,101 @@
import { renderToStaticMarkup } from 'react-dom/server'
import { describe, expect, it, vi } from 'vitest'
import { ProductionBindingsView } from './ProductionBindingsSection'
import type { ProductionBindings } from './productionBindings'
const bindings = (overrides: Partial<ProductionBindings> = {}): ProductionBindings => ({
internal_kb_id: '0b217e8a-2a65-40a3-ab8f-34ac5c7a6f05',
external_kb_id: '9a1e5c11-0000-4000-8000-000000000002',
internal_kb_name: '对内知识库-wiki',
external_kb_name: '对外知识库-wiki',
updated_by: 'yaozhanghao',
production_base_url: 'https://weknora-test.dobot.cn',
production_connected: true,
production_message: '',
legacy_scope: '文档级发布(单篇文档写生产)· 旧路径',
whole_target: {
configured: true,
source: '部署环境 LIBRARY_PRODUCTION_*',
tenant_id: 10000,
internal_kb_id: 'a8e9c8ad-411c-4c53-8421-599d29e9b179',
external_kb_id: '2e980686-2233-4c4e-b5a6-8b0069477b8e',
database: 'pgm-wz9ohr9gk919pr1q.pg.rds.aliyuncs.com:5432/weknora_test',
elastic_index: 'weknora_test_chunks',
object_bucket: 'dobot-weknora-test',
missing_reason: '',
},
...overrides,
})
const view = (value: ProductionBindings | null, admin = true) =>
renderToStaticMarkup(
<ProductionBindingsView
bindings={value}
options={[]}
internalDraft={value?.internal_kb_id ?? ''}
externalDraft={value?.external_kb_id ?? ''}
isAdmin={admin}
loading={false}
saving={false}
error={null}
message=""
dirty={false}
sameSelection={false}
onInternalChange={vi.fn()}
onExternalChange={vi.fn()}
onSave={vi.fn()}
onReload={vi.fn()}
/>
)
describe('ProductionBindingsView', () => {
it('把实际整库目标(完整 ID + 端点)显示出来,而不是只显示可编辑的文档级绑定', () => {
const html = view(bindings())
expect(html).toContain('整库发布目标(实际写入,只读)')
// 完整 ID 必须可读:截断会让操作者核对不了 a8e9c8ad 与 0b217e8a 的区别。
expect(html).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179')
expect(html).toContain('2e980686-2233-4c4e-b5a6-8b0069477b8e')
expect(html).toContain('pgm-wz9ohr9gk919pr1q.pg.rds.aliyuncs.com:5432/weknora_test')
expect(html).toContain('weknora_test_chunks')
expect(html).toContain('dobot-weknora-test')
expect(html).toContain('部署环境 LIBRARY_PRODUCTION_*')
})
it('旧文档级绑定标成旧路径,不再声称"发布写入此库"', () => {
const html = view(bindings())
expect(html).toContain('文档级发布(单篇文档写生产)· 旧路径')
expect(html).toContain('仅文档级发布读取这组绑定;整库发布(封存/同步/恢复)不读它。')
expect(html).toContain('对内文档级目标')
expect(html).not.toContain('发布写入此库')
})
it('整库目标缺配置:点名缺哪个环境变量,且不拿文档级绑定顶上去', () => {
const html = view(
bindings({
whole_target: {
configured: false,
source: '部署环境 LIBRARY_PRODUCTION_*',
tenant_id: 0,
internal_kb_id: '',
external_kb_id: '',
database: '(未配置)',
elastic_index: '',
object_bucket: '',
missing_reason: '未配置:LIBRARY_PRODUCTION_DATABASE_URL、LIBRARY_PRODUCTION_KB_INTERNAL',
},
})
)
expect(html).toContain('LIBRARY_PRODUCTION_DATABASE_URL')
expect(html).toContain('整库目标配置不全')
// 文档级绑定仍然显示,但必须仍是"文档级"身份,不能冒充整库目标。
expect(html).toContain('对内文档级目标')
})
it('非管理员看得到目标,但没有保存入口', () => {
const html = view(bindings(), false)
expect(html).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179')
expect(html).not.toContain('保存文档级目标')
expect(html).toContain('仅管理员可修改文档级发布目标')
})
})
+196 -88
View File
@@ -5,8 +5,182 @@ import { fetchProductionBindingOptions, fetchProductionBindings, saveProductionB
import type { ProductionBindings } from './productionBindings'
import type { KnowledgeBaseOption } from './routingBindings'
// 「生产知识库绑定」:发布链路的 源库 → 目标库 映射。这里选的是发布要写入的
// 生产(目标)知识库,与上面的「测试知识库绑定」(测试端收件库)是两件事。
// 「生产知识库绑定」:这里同时涉及两条独立路径,页面必须分开写清楚:
// 1) 整库发布目标(封存/同步/恢复真正写入的库与端点)——来自部署配置,只读;
// 2) 文档级绑定(旧流程:单篇文档发布写哪个库)——可在这里编辑。
// 把 2 写成"发布写入此库"会让操作者以为整库发布会写进 0b217e8a 这类 2000+ 文档库。
export interface ProductionBindingsViewProps {
bindings: ProductionBindings | null
options: KnowledgeBaseOption[]
internalDraft: string
externalDraft: string
isAdmin: boolean
loading: boolean
saving: boolean
error: string | null
message: string
dirty: boolean
sameSelection: boolean
onInternalChange: (next: string) => void
onExternalChange: (next: string) => void
onSave: () => void
onReload: () => void
}
export function ProductionBindingsView(props: ProductionBindingsViewProps) {
const {
bindings,
options,
internalDraft,
externalDraft,
isAdmin,
loading,
saving,
error,
message,
dirty,
sameSelection,
onInternalChange,
onExternalChange,
onSave,
onReload,
} = props
const optionLabel = (base: KnowledgeBaseOption) => `${base.name}(${base.knowledge_count} 文档)`
const renderSide = (label: string, boundID: string, boundName: string) => (
<div className={`kb-binding-side ${boundID ? 'bound' : 'warn'}`}>
<b>{label}</b>
<span>
{boundID ? boundName || `${boundID.slice(0, 8)}…` : '未绑定 · 按同 ID 语义发布(目标端须存在同 ID 库)'}
</span>
<small>{boundID ? '文档级绑定 · 只影响单篇文档发布,不改整库目标' : '未绑定:沿用测试库 ID'}</small>
</div>
)
const whole = bindings?.whole_target
const wholeRows: [string, string][] = whole
? [
['对内目标知识库', whole.internal_kb_id || '未配置'],
['对外目标知识库', whole.external_kb_id || '未配置'],
['目标数据库', whole.database || '—'],
['检索索引', whole.elastic_index || '—'],
['对象存储桶', whole.object_bucket || '—'],
['租户', whole.tenant_id ? String(whole.tenant_id) : '—'],
['配置来源', whole.source || '—'],
]
: []
const editorFor = (value: string, onChange: (next: string) => void, placeholder: string) =>
options.length > 0 ? (
<select value={value} onChange={(event) => onChange(event.target.value)} aria-label={placeholder}>
<option value="">{placeholder}</option>
{options.map((base) => (
<option key={base.id} value={base.id}>
{optionLabel(base)}
</option>
))}
</select>
) : (
<input
value={value}
onChange={(event) => onChange(event.target.value.trim())}
placeholder={`${placeholder}(输入生产知识库 ID)`}
aria-label={placeholder}
/>
)
return (
<section className="settings-panel wide" id="settings-production-bindings">
<div className="settings-section-heading">
<div>
<h2>生产知识库绑定</h2>
<p>
这里有两件不同的事,页面分开显示:<strong>整库发布目标</strong>(封存/同步/恢复真正写入的目标,
由部署配置决定,只读)与<strong>文档级绑定</strong>(旧流程:单篇文档发布写哪个库,可在这里改)。
</p>
</div>
<button onClick={onReload} disabled={loading}>
{loading ? '读取中…' : '重新读取'}
</button>
</div>
{error ? (
<p className="error-message" role="alert">
{error}
</p>
) : null}
{bindings ? (
<div className="stack" id="settings-whole-library-target">
<h3>整库发布目标(实际写入,只读)</h3>
{whole?.configured === false ? (
<p className="hint">
整库目标配置不全:{whole.missing_reason}。整库封存/同步/恢复在补齐部署配置前会被拒绝,
页面不会用下面的文档级绑定顶上。
</p>
) : null}
{wholeRows.length ? (
<dl className="storage-settings-grid">
{wholeRows.map(([label, value]) => (
<div key={label}>
<dt>{label}</dt>
<dd>
<code>{value}</code>
</dd>
</div>
))}
</dl>
) : null}
<p className="hint">
这份目标来自部署环境,不在页面上编辑;改它需要改部署配置并重启 api/worker,不会由本页保存。
</p>
</div>
) : null}
{bindings ? (
<div className="stack">
<h3>{bindings.legacy_scope || '文档级绑定(旧路径)'}</h3>
<p className="hint">仅文档级发布读取这组绑定;整库发布(封存/同步/恢复)不读它。</p>
<div className="kb-binding-grid">
{renderSide('对内文档级目标', bindings.internal_kb_id, bindings.internal_kb_name)}
{renderSide('对外文档级目标', bindings.external_kb_id, bindings.external_kb_name)}
</div>
</div>
) : null}
{bindings && !bindings.production_connected ? (
<p className="hint">
生产端核对不可用:{bindings.production_message}。
{bindings.production_base_url ? `(生产地址:${bindings.production_base_url})` : ''}
</p>
) : null}
{bindings && bindings.production_connected && bindings.production_base_url ? (
<p className="hint">
生产端:{bindings.production_base_url}
{bindings.updated_by ? ` · 最近修改:${bindings.updated_by}` : ''}
</p>
) : null}
{isAdmin && bindings ? (
<div className="kb-binding-editor">
<label>
对内文档级目标 {editorFor(internalDraft, onInternalChange, '选择对内文档级目标库')}
</label>
<label>
对外文档级目标 {editorFor(externalDraft, onExternalChange, '选择对外文档级目标库')}
</label>
<p className="hint">保存时只改文档级绑定;整库发布目标由部署配置决定,本页不改它。</p>
<p className="hint">保存时会向生产端核对所选的库是否真实存在;对不上会拒收,生产端不可达则只提示。</p>
{sameSelection ? <p className="error-message">对内与对外必须绑定不同的生产知识库</p> : null}
<button onClick={onSave} disabled={saving || !dirty || sameSelection}>
{saving ? '正在核对并保存…' : '保存文档级目标'}
</button>
{message ? (
<p className="success-message" role="status">
{message}
</p>
) : null}
</div>
) : null}
{!isAdmin ? <p className="hint">仅管理员可修改文档级发布目标。</p> : null}
</section>
)
}
// 容器:读取 + 保存。整库目标只读展示,保存动作只写文档级绑定。
export function ProductionBindingsSection() {
const auth = useOptionalAuth()
const isAdmin = auth?.identity?.role === 'admin'
@@ -56,8 +230,8 @@ export function ProductionBindingsSection() {
const where = `对内 ${next.internal_kb_name || next.internal_kb_id.slice(0, 8) || '未绑定'} · 对外 ${next.external_kb_name || next.external_kb_id.slice(0, 8) || '未绑定'}`
setMessage(
next.production_connected
? `发布目标已更新:${where}。之后新发布的“对内/对外”内容会写入这两个生产库(已发布过的条目仍回到原目标)。`
: `发布目标已保存:${where};但当前核对不了生产端(${next.production_message}),下次发布前请确认。`
? `文档级发布目标已更新:${where}。只影响单篇文档发布;整库发布写的是上面那份「整库发布目标」。`
: `文档级发布目标已保存:${where};但当前核对不了生产端(${next.production_message}),下次发布前请确认。`
)
} catch (cause) {
setError(cause instanceof Error ? cause.message : '保存生产知识库绑定失败')
@@ -66,89 +240,23 @@ export function ProductionBindingsSection() {
}
}
const optionLabel = (base: KnowledgeBaseOption) => `${base.name}(${base.knowledge_count} 文档)`
const renderSide = (label: string, boundID: string, boundName: string) => (
<div className={`kb-binding-side ${boundID ? 'bound' : 'warn'}`}>
<b>{label}</b>
<span>
{boundID ? boundName || `${boundID.slice(0, 8)}…` : '未绑定 · 按同 ID 语义发布(目标端须存在同 ID 库)'}
</span>
<small>{boundID ? 'UI 绑定 · 发布写入此库' : '未绑定:沿用测试库 ID'}</small>
</div>
)
const editorFor = (value: string, onChange: (next: string) => void, placeholder: string) =>
options.length > 0 ? (
<select value={value} onChange={(event) => onChange(event.target.value)} aria-label={placeholder}>
<option value="">{placeholder}</option>
{options.map((base) => (
<option key={base.id} value={base.id}>
{optionLabel(base)}
</option>
))}
</select>
) : (
<input
value={value}
onChange={(event) => onChange(event.target.value.trim())}
placeholder={`${placeholder}(输入生产知识库 ID)`}
aria-label={placeholder}
/>
)
return (
<section className="settings-panel wide" id="settings-production-bindings">
<div className="settings-section-heading">
<div>
<h2>生产知识库绑定</h2>
<p>
发布链路的“源库 → 目标库”:新发布的对内 / 对外内容按这里的绑定写入生产 WeKnora
的对应知识库。留空表示未绑定,按同 ID 语义发布。
</p>
</div>
<button onClick={() => void load()} disabled={loading}>
{loading ? '读取中…' : '重新读取'}
</button>
</div>
{error ? (
<p className="error-message" role="alert">
{error}
</p>
) : null}
{bindings ? (
<div className="kb-binding-grid">
{renderSide('对内生产知识库', bindings.internal_kb_id, bindings.internal_kb_name)}
{renderSide('对外生产知识库', bindings.external_kb_id, bindings.external_kb_name)}
</div>
) : null}
{bindings && !bindings.production_connected ? (
<p className="hint">
生产端核对不可用:{bindings.production_message}。
{bindings.production_base_url ? `(生产地址:${bindings.production_base_url})` : ''}
</p>
) : null}
{bindings && bindings.production_connected && bindings.production_base_url ? (
<p className="hint">
生产端:{bindings.production_base_url}
{bindings.updated_by ? ` · 最近修改:${bindings.updated_by}` : ''}
</p>
) : null}
{isAdmin && bindings ? (
<div className="kb-binding-editor">
<label>对内目标库 {editorFor(internalDraft, setInternalDraft, '选择对内生产知识库')}</label>
<label>对外目标库 {editorFor(externalDraft, setExternalDraft, '选择对外生产知识库')}</label>
<p className="hint">保存时会向生产端核对所选的库是否真实存在;对不上会拒收,生产端不可达则只提示。</p>
{sameSelection ? <p className="error-message">对内与对外必须绑定不同的生产知识库</p> : null}
<button onClick={() => void save()} disabled={saving || !dirty || sameSelection}>
{saving ? '正在核对并保存…' : '保存发布目标'}
</button>
{message ? (
<p className="success-message" role="status">
{message}
</p>
) : null}
</div>
) : null}
{!isAdmin ? <p className="hint">仅管理员可修改发布目标。</p> : null}
</section>
<ProductionBindingsView
bindings={bindings}
options={options}
internalDraft={internalDraft}
externalDraft={externalDraft}
isAdmin={isAdmin}
loading={loading}
saving={saving}
error={error}
message={message}
dirty={dirty}
sameSelection={sameSelection}
onInternalChange={setInternalDraft}
onExternalChange={setExternalDraft}
onSave={() => void save()}
onReload={() => void load()}
/>
)
}
}
+9 -1
View File
@@ -27,6 +27,7 @@ import {
} from './librarySnapshots'
import { useOptionalAuth } from './auth'
import './releaseTargets.css'
import { RestoreVerificationSection } from './RestoreVerificationSection'
// 发布管理:按已确认稿实现——顶部两个知识库页签,下面是**一个**作业区。
//
@@ -260,6 +261,7 @@ export function jobKindText(kind?: string): string {
if (kind === 'sync') return '生产同步'
if (kind === 'restore') return '按版本恢复'
if (kind === 'baseline') return '建立目标基线'
if (kind === 'verify') return '恢复验证'
return kind ? kind : '—'
}
@@ -281,7 +283,9 @@ export function publishStateText(job?: LibraryJobRecord): string {
? '恢复进行中(目标在维护中)'
: job.kind === 'baseline'
? '建立基线进行中(只读捕获目标,不写内容)'
: '同步进行中(目标在维护中)'
: job.kind === 'verify'
? '恢复验证进行中(目标在维护中)'
: '同步进行中(目标在维护中)'
}
if (job.state === 'succeeded') return `${jobKindText(job.kind)}已完成 · ${stamp(job.finished_at ?? job.updated_at)}`
if (job.state === 'failed') return `${jobKindText(job.kind)}失败 · ${stamp(job.finished_at ?? job.updated_at)}`
@@ -718,6 +722,7 @@ function ProtectionCard({
targetSealed,
sync,
baseline,
scope,
onOpen,
}: {
production: LibraryOverview['production'] | null
@@ -730,6 +735,7 @@ function ProtectionCard({
targetSealed: boolean
sync: ReleaseSyncActions
baseline: ReleaseBaselineActions
scope: Scope
onOpen: () => void
}) {
const backupKnown = prebackup.state === 'known' && prebackup.version
@@ -798,6 +804,7 @@ function ProtectionCard({
<dt>恢复条件</dt>
<dd>{gate.reasons.length === 0 ? '已满足' : `缺 ${gate.reasons.length} 项`}</dd>
</dl>
<RestoreVerificationSection scope={scope} isAdmin={sync.canSign} />
{gate.reasons.length > 0 ? (
<ul className="rel-gate" aria-label="恢复与启动条件">
{gate.reasons.map((reason) => (
@@ -880,6 +887,7 @@ function ReadyBody({
targetSealed={Boolean(snapshot)}
sync={sync}
baseline={baseline}
scope={view.scope}
onOpen={onOpen}
/>
</div>
+103
View File
@@ -0,0 +1,103 @@
import { renderToStaticMarkup } from 'react-dom/server'
import { describe, expect, it } from 'vitest'
import { RestoreVerificationView } from './RestoreVerificationSection'
import type { LibraryVerificationView } from './restoreVerification'
const target = {
instance: 'weknora-test',
project: 'weknora',
internal_kb: 'a8e9c8ad-411c-4c53-8421-599d29e9b179',
external_kb: '2e980686-2233-4c4e-b5a6-8b0069477b8e',
index: 'weknora_test_chunks',
s3_bucket: 'dobot-weknora-test',
tenant_id: 10000,
database_configured: true,
}
const view = (overrides: Partial<LibraryVerificationView> = {}): LibraryVerificationView => ({
scope: 'internal',
target,
verified: false,
reason: '还没有对当前目标做过恢复验证',
latest: null,
history: [],
...overrides,
})
const render = (value: LibraryVerificationView | null, isAdmin = true, pending = false, error: string | null = null) =>
renderToStaticMarkup(
<RestoreVerificationView
view={value}
scope="internal"
isAdmin={isAdmin}
pending={pending}
error={error}
notice=""
onRun={() => undefined}
onReload={() => undefined}
/>
)
describe('RestoreVerificationView', () => {
it('没有证据时显示"还没有验证记录"并给出下一步,不显示已验证', () => {
const html = render(view())
expect(html).toContain('还没有验证记录')
expect(html).toContain('跑一次恢复验证')
expect(html).not.toContain('已验证(当前目标)')
})
it('管理员能看到发起按钮,工程师只看到证据与原因', () => {
expect(render(view(), true)).toContain('跑一次恢复验证')
const engineer = render(view(), false)
expect(engineer).not.toContain('跑一次恢复验证')
expect(engineer).toContain('仅管理员可发起')
})
it('通过后显示已验证、快照与完整目标', () => {
const html = render(
view({
verified: true,
reason: '',
latest: {
id: 'libverify-internal-job1',
scope: 'internal',
status: 'passed',
snapshot_id: '0f9b39a16738',
job_id: 'libverify-internal-1',
started_at: '2026-11-03T01:00:00Z',
finished_at: '2026-11-03T01:02:00Z',
},
})
)
expect(html).toContain('已验证(当前目标)')
expect(html).toContain('0f9b39a16738')
expect(html).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179')
expect(html).toContain('weknora_test_chunks')
})
it('失败时把失败原因写出来,按钮仍可重试', () => {
const html = render(
view({
latest: {
id: 'libverify-internal-job2',
scope: 'internal',
status: 'failed',
snapshot_id: 'snap-2',
job_id: 'libverify-internal-2',
failure: '恢复后内容仍与预备份不一致',
started_at: '2026-11-03T01:00:00Z',
finished_at: '2026-11-03T01:05:00Z',
},
})
)
expect(html).toContain('最近一次未通过')
expect(html).toContain('恢复后内容仍与预备份不一致')
expect(html).toContain('跑一次恢复验证')
})
it('投递被拒时显示后端逐条原因', () => {
const html = render(view(), true, false, '恢复验证被拒绝:生产写入闸口关闭')
expect(html).toContain('生产写入闸口关闭')
})
})
+140
View File
@@ -0,0 +1,140 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import {
enqueueLibraryVerification,
fetchLibraryVerification,
verificationStatusText,
verificationTargetLines,
} from './restoreVerification'
import type { LibraryVerificationView } from './restoreVerification'
// 「恢复验证」面板:同步闸口认的"生产可恢复"证据只能由这里发起的一次真实演练产生。
//
// 为什么要有这个入口:以前只有 CLI(现为 internal/libraryverify 的薄封装)能跑演练,
// 页面只能看到一个 env 开关。运维在页面上看到"缺 1 项",却没有任何可点的下一步。
// 这里把"跑一次真实演练"变成正常流程:它会停写入者、写 canary、按预备份恢复,
// 全过程由 worker 执行并登记真实终态;页面只显示结果,不提供手填开关。
export interface RestoreVerificationViewProps {
view: LibraryVerificationView | null
scope: string
isAdmin: boolean
pending: boolean
error: string | null
notice: string
onRun: () => void
onReload: () => void
}
const stampText = (value: string): string => {
if (!value) return '—'
const parsed = new Date(value)
return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString()
}
export function RestoreVerificationView(props: RestoreVerificationViewProps) {
const { view, scope, isAdmin, pending, error, notice, onRun, onReload } = props
const status = verificationStatusText(view)
const latest = view?.latest ?? null
const running = Boolean(view?.running)
const tone = view?.verified ? 'green' : latest?.status === 'failed' ? 'red' : 'gray'
return (
<section className="rel-verification" aria-label="恢复验证">
<header className="rel-between">
<h3>恢复验证(当前整库目标)</h3>
<span className={`rel-pill ${tone}`}>{status}</span>
</header>
{view ? (
<ul className="rel-target-lines" aria-label="恢复验证目标">
{verificationTargetLines(view.target, scope).map((line) => (
<li key={line}>{line}</li>
))}
</ul>
) : null}
<p className="rel-hint">{view?.reason || '还没有读到验证状态。'}</p>
{latest ? (
<p className="rel-hint">
最近一次:{latest.status === 'passed' ? '通过' : '未通过'} · 快照 {latest.snapshot_id.slice(0, 12) || '—'} ·{' '}
{stampText(latest.finished_at)} · 作业 {latest.job_id}
</p>
) : null}
{latest?.failure ? <p className="rel-warn">未通过原因:{latest.failure}</p> : null}
{error ? <p className="rel-warn">{error}</p> : null}
{notice ? <p className="rel-hint">{notice}</p> : null}
<div className="rel-drawer-actions">
{isAdmin ? (
<button type="button" className="rel-secondary" onClick={onRun} disabled={pending || running}>
{pending ? '投递中…' : running ? '验证执行中…' : '跑一次恢复验证(维护隔离)'}
</button>
) : (
<span className="rel-hint">恢复验证仅管理员可发起;工程师可以看到证据与原因。</span>
)}
<button type="button" className="rel-secondary" onClick={onReload} disabled={pending}>
刷新验证状态
</button>
</div>
<p className="rel-hint">
验证会停住目标写入者、写一个隔离 canary 证明校验有效、再按预备份恢复并逐类复核;
通过后证据按当前目标登记,换库/换索引/换桶后自动失效。
</p>
</section>
)
}
// 容器:挂载即读一次,之后每 5 秒刷新(验证作业在 worker 里跑,页面只读证据)。
export function RestoreVerificationSection({ scope, isAdmin }: { scope: string; isAdmin: boolean }) {
const [view, setView] = useState<LibraryVerificationView | null>(null)
const [pending, setPending] = useState(false)
const [error, setError] = useState<string | null>(null)
const [notice, setNotice] = useState('')
const mounted = useRef(true)
const reload = useCallback(
(signal?: AbortSignal) => {
fetchLibraryVerification(scope, signal)
.then((next) => {
if (mounted.current && next) setView(next)
})
.catch(() => undefined)
},
[scope]
)
useEffect(() => {
mounted.current = true
const controller = new AbortController()
reload(controller.signal)
const timer = setInterval(() => reload(controller.signal), 5000)
return () => {
mounted.current = false
controller.abort()
clearInterval(timer)
}
}, [reload])
const onRun = useCallback(() => {
setPending(true)
setError(null)
setNotice('')
enqueueLibraryVerification(scope)
.then((accepted) => {
setNotice(`已投递恢复验证作业 ${accepted.job_id};进度在作业进度里可见,完成后这里显示证据。`)
reload()
})
.catch((err: unknown) => setError(err instanceof Error ? err.message : '投递恢复验证失败'))
.finally(() => setPending(false))
}, [scope, reload])
return (
<RestoreVerificationView
view={view}
scope={scope}
isAdmin={isAdmin}
pending={pending}
error={error}
notice={notice}
onRun={onRun}
onReload={() => reload()}
/>
)
}
+2 -1
View File
@@ -332,7 +332,8 @@ describe('SystemSettingsView', () => {
// 发布目标(源库 → 目标库)是独立分区,与测试端收件库分开。
const target = view('production-bindings')
expect(target).toContain('id="settings-production-bindings"')
expect(target).toContain('新发布的对内 / 对外内容按这里的绑定写入生产 WeKnora 的对应知识库')
expect(target).toContain('整库发布目标')
expect(target).toContain('文档级绑定')
})
it('does not show healthy simulation when the settings data failed to load', () => {
const html = renderToStaticMarkup(
+3
View File
@@ -12,6 +12,8 @@ export interface BackupSchedule {
last_applied_summary: string
host_configured: boolean
host_target: string
// host_endpoint 是这条别名真实拨号的 host:port(来自挂载的 ssh 配置)。
host_endpoint: string
host_reachable: boolean
host_error: string
host_schedule_enabled: boolean
@@ -70,6 +72,7 @@ function scheduleProjection(item: Record<string, unknown>): BackupSchedule {
last_applied_summary: asString(item.last_applied_summary),
host_configured: item.host_configured === true,
host_target: asString(item.host_target),
host_endpoint: asString(item.host_endpoint),
host_reachable: item.host_reachable === true,
host_error: asString(item.host_error),
host_schedule_enabled: item.host_schedule_enabled === true,
+1 -1
View File
@@ -567,7 +567,7 @@ export interface LibraryStageRow {
export interface LibraryJobRecord {
id: string
scope: string
kind: 'seal' | 'sync' | 'restore' | 'baseline'
kind: 'seal' | 'sync' | 'restore' | 'baseline' | 'verify'
state: 'pending' | 'running' | 'succeeded' | 'failed' | 'cancelled'
stage: string
request_digest: string
+34
View File
@@ -11,6 +11,22 @@ export interface ProductionBindings {
production_base_url: string
production_connected: boolean
production_message: string
// legacy_scope 标明这条记录属于旧文档级发布路径;整库发布不看它。
legacy_scope: string
// whole_target 是整库发布真正写入的目标(来自部署配置),只读。
whole_target: LibraryPublishTarget
}
export interface LibraryPublishTarget {
configured: boolean
source: string
tenant_id: number
internal_kb_id: string
external_kb_id: string
database: string
elastic_index: string
object_bucket: string
missing_reason: string
}
const record = (value: unknown): value is Record<string, unknown> =>
@@ -26,6 +42,24 @@ function projection(item: Record<string, unknown>): ProductionBindings {
production_base_url: typeof item.production_base_url === 'string' ? item.production_base_url : '',
production_connected: item.production_connected === true,
production_message: typeof item.production_message === 'string' ? item.production_message : '',
legacy_scope: typeof item.legacy_scope === 'string' ? item.legacy_scope : '',
whole_target: wholeTarget(item.whole_target),
}
}
// wholeTarget 只读投影:缺字段就说未配置,不拿可编辑的文档级绑定冒充整库目标。
function wholeTarget(value: unknown): LibraryPublishTarget {
const item = record(value) ? value : {}
return {
configured: item.configured === true,
source: typeof item.source === 'string' ? item.source : '',
tenant_id: typeof item.tenant_id === 'number' ? item.tenant_id : 0,
internal_kb_id: typeof item.internal_kb_id === 'string' ? item.internal_kb_id : '',
external_kb_id: typeof item.external_kb_id === 'string' ? item.external_kb_id : '',
database: typeof item.database === 'string' ? item.database : '',
elastic_index: typeof item.elastic_index === 'string' ? item.elastic_index : '',
object_bucket: typeof item.object_bucket === 'string' ? item.object_bucket : '',
missing_reason: typeof item.missing_reason === 'string' ? item.missing_reason : '',
}
}
+93
View File
@@ -0,0 +1,93 @@
import { describe, expect, it } from 'vitest'
import {
parseVerificationRecord,
parseVerificationView,
verificationStatusText,
verificationTargetLines,
} from './restoreVerification'
import type { LibraryVerificationView } from './restoreVerification'
const target = {
instance: 'weknora-test',
project: 'weknora',
internal_kb: 'a8e9c8ad-411c-4c53-8421-599d29e9b179',
external_kb: '2e980686-2233-4c4e-b5a6-8b0069477b8e',
index: 'weknora_test_chunks',
s3_bucket: 'dobot-weknora-test',
tenant_id: 10000,
database_configured: true,
}
const passed = {
id: 'libverify-internal-job1',
scope: 'internal',
status: 'passed',
snapshot_id: '0f9b39a16738',
job_id: 'libverify-internal-1',
started_at: '2026-11-03T01:00:00Z',
finished_at: '2026-11-03T01:02:00Z',
}
describe('restoreVerification parsing', () => {
it('只把后端明确 true 当作已验证', () => {
const view = parseVerificationView({ scope: 'internal', target, verified: 'true', reason: '', history: [] })
expect(view?.verified).toBe(false)
})
it('保留失败证据的失败原因', () => {
const record = parseVerificationRecord({ ...passed, status: 'failed', failure: '恢复后内容仍与预备份不一致' })
expect(record?.status).toBe('failed')
expect(record?.failure).toBe('恢复后内容仍与预备份不一致')
})
it('未知状态一律丢弃,不猜成通过', () => {
expect(parseVerificationRecord({ ...passed, status: 'running' })).toBeNull()
})
it('没有目标信息的响应视为无效', () => {
expect(parseVerificationView({ scope: 'internal', verified: true })).toBeNull()
})
})
describe('restoreVerification display', () => {
const base: LibraryVerificationView = {
scope: 'internal',
target,
verified: true,
reason: '',
latest: {
id: passed.id,
scope: 'internal',
status: 'passed',
snapshot_id: passed.snapshot_id,
job_id: passed.job_id,
started_at: passed.started_at,
finished_at: passed.finished_at,
},
history: [],
}
it('整库目标显示完整 KB ID 与索引/桶,不缩写成"2074 文档库"', () => {
const lines = verificationTargetLines(target, 'internal')
expect(lines.join(' | ')).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179')
expect(lines.join(' | ')).toContain('weknora_test_chunks')
expect(lines.join(' | ')).toContain('dobot-weknora-test')
})
it('对外 scope 显示对外目标', () => {
expect(verificationTargetLines(target, 'external').join(' | ')).toContain('2e980686-2233-4c4e-b5a6-8b0069477b8e')
})
it('状态文案区分已验证 / 最近失败 / 无记录', () => {
expect(verificationStatusText(base)).toBe('已验证(当前目标)')
expect(
verificationStatusText({
...base,
verified: false,
latest: { ...base.latest!, status: 'failed' },
})
).toBe('最近一次未通过')
expect(verificationStatusText({ ...base, verified: false, latest: null })).toBe('还没有验证记录')
})
})
+158
View File
@@ -0,0 +1,158 @@
// 恢复验证的页面数据层:读证据 + 投递真实演练。
//
// 关键约定:verified 只由后端证据回答(当前目标 + scope 的最近一次真实演练)。
// 页面不提供任何"我确认已验证"的开关——那正是旧实现的问题(一个可手填的 env)。
export type LibraryVerificationStatus = 'passed' | 'failed'
export interface LibraryVerificationRecord {
id: string
scope: string
status: LibraryVerificationStatus
snapshot_id: string
job_id: string
failure?: string
started_at: string
finished_at: string
}
export interface LibraryVerificationTarget {
instance: string
project: string
internal_kb: string
external_kb: string
index: string
s3_bucket: string
tenant_id: number
database_configured: boolean
}
export interface LibraryVerificationView {
scope: string
target: LibraryVerificationTarget
verified: boolean
reason: string
latest?: LibraryVerificationRecord | null
history: LibraryVerificationRecord[]
running?: boolean
job_id?: string
}
function record(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null
}
function text(value: unknown): string {
return typeof value === 'string' ? value : ''
}
function bool(value: unknown): boolean {
return value === true
}
export function parseVerificationTarget(value: unknown): LibraryVerificationTarget | null {
if (!record(value)) return null
return {
instance: text(value.instance),
project: text(value.project),
internal_kb: text(value.internal_kb),
external_kb: text(value.external_kb),
index: text(value.index),
s3_bucket: text(value.s3_bucket),
tenant_id: typeof value.tenant_id === 'number' ? value.tenant_id : 0,
database_configured: bool(value.database_configured),
}
}
export function parseVerificationRecord(value: unknown): LibraryVerificationRecord | null {
if (!record(value)) return null
const status = text(value.status)
if (status !== 'passed' && status !== 'failed') return null
return {
id: text(value.id),
scope: text(value.scope),
status,
snapshot_id: text(value.snapshot_id),
job_id: text(value.job_id),
failure: text(value.failure) || undefined,
started_at: text(value.started_at),
finished_at: text(value.finished_at),
}
}
export function parseVerificationView(value: unknown): LibraryVerificationView | null {
if (!record(value)) return null
const target = parseVerificationTarget(value.target)
if (!target) return null
const history = Array.isArray(value.history)
? value.history.map(parseVerificationRecord).filter((item): item is LibraryVerificationRecord => item !== null)
: []
const latest = parseVerificationRecord(value.latest) ?? null
return {
scope: text(value.scope),
target,
// 只有后端明确说 true 才算已验证:任何解析失败都退回"没有证据"。
verified: bool(value.verified),
reason: text(value.reason),
latest,
history,
running: bool(value.running),
job_id: text(value.job_id) || undefined,
}
}
export async function fetchLibraryVerification(
scope: string,
signal?: AbortSignal
): Promise<LibraryVerificationView | null> {
const response = await fetch(`/api/v1/library/verifications?scope=${encodeURIComponent(scope)}`, {
headers: { Accept: 'application/json' },
signal,
})
const value = (await response.json().catch(() => null)) as unknown
return parseVerificationView(value)
}
export interface LibraryVerificationAccepted {
job_id: string
scope: string
}
// 投递一次真实恢复验证。失败时把后端的逐条拒绝原因原样显示,
// 不把它包装成"稍后再试"——否则用户不知道是闸口、维护还是目标不可达。
export async function enqueueLibraryVerification(scope: string): Promise<LibraryVerificationAccepted> {
const response = await fetch('/api/v1/library/verifications', {
method: 'POST',
headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
body: JSON.stringify({ scope }),
})
const value = (await response.json().catch(() => null)) as unknown
if (!response.ok) {
if (response.status === 403) throw new Error('恢复验证仅管理员可发起。')
if (record(value) && Array.isArray(value.reasons)) {
const reasons = value.reasons.map(text).filter(Boolean)
if (reasons.length > 0) throw new Error(`恢复验证被拒绝:${reasons.join(';')}`)
}
if (record(value) && text(value.error)) throw new Error(text(value.error))
throw new Error(`投递恢复验证失败(HTTP ${response.status})`)
}
if (!record(value) || !record(value.job)) throw new Error('恢复验证作业响应无效')
return { job_id: text(value.job.id), scope: text(value.job.scope) }
}
// 目标展示:整库目标必须显示完整 ID 与端点,避免"看着像 2074 文档库"这类误读。
export function verificationTargetLines(target: LibraryVerificationTarget, scope: string): string[] {
const kb = scope === 'internal' ? target.internal_kb : target.external_kb
return [
`实例 ${target.instance || '未配置'} · 项目 ${target.project || '未配置'}`,
`知识库 ${kb || '未配置'}`,
`索引 ${target.index || '未配置'} · 桶 ${target.s3_bucket || '未配置'}`,
`租户 ${target.tenant_id || '未配置'} · 数据库${target.database_configured ? '已配置' : '未配置'}`,
]
}
export function verificationStatusText(view: LibraryVerificationView | null): string {
if (!view) return '还没有读到验证状态'
if (view.verified) return '已验证(当前目标)'
if (view.latest) return view.latest.status === 'failed' ? '最近一次未通过' : '未通过'
return '还没有验证记录'
}