From 5e1482e349e23728f8f2f1aeb770422eba2e5b5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Tue, 3 Nov 2026 10:14:08 +0800 Subject: [PATCH] =?UTF-8?q?feat(library):=20=E6=81=A2=E5=A4=8D=E9=AA=8C?= =?UTF-8?q?=E8=AF=81=E6=88=90=E4=B8=BA=E5=8F=AF=E9=A1=B5=E9=9D=A2=E5=8F=91?= =?UTF-8?q?=E8=B5=B7=E7=9A=84=E7=9C=9F=E5=AE=9E=E4=BD=9C=E4=B8=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 根因：生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关， 页面没有任何入口去真跑一次恢复；CLI（librarydrill）又不是 worker 能复用的实现， 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动： - 共享实现 internal/libraryverify（维护隔离→捕获→基线校验→canary→恢复→复核→关窗口）， cmd/librarydrill 改为薄封装，CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业：kind + VerifyStages(capture/verify/canary/restore/finalize)， 与 sync/restore/baseline 共用同一把写互斥；迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications（按 target_id 绑定，换库/索引/桶旧证据自动失效， 同 target+scope+job 幂等）。 - worker：admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现，阶段实时写 library_job_stages，结束后写真实终态；通过写 passed 证据，失败写 failed+原因（不伪成功）； 中断接管按同一作业 id 收掉维护窗口。 - API：POST/GET /api/v1/library/verifications（管理员；闸口关闭/维护中拒绝；返回完整 整库目标、最近证据、失败原因），OpenAPI 同步。 - 闸口：同步/恢复/续作的证据判断改为读当前目标证据；历史 env 仅在证据缺失时兜底， 证据存在时以证据为准。 - 页面：生产保护与恢复面板加「恢复验证」块（完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询），作业类型补 verify 文案。 测试：make verify 16/16 全绿（go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定）； 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。 --- api/openapi.yaml | 87 ++++++ cmd/api/main.go | 2 + cmd/librarydrill/canary.go | 21 -- cmd/librarydrill/main.go | 215 +------------ cmd/worker/library_handlers.go | 8 +- cmd/worker/library_release_handlers.go | 5 + cmd/worker/library_role_test.go | 6 +- cmd/worker/library_verify.go | 188 ++++++++++++ cmd/worker/library_verify_test.go | 53 ++++ cmd/worker/main.go | 4 + internal/appconfig/library_role.go | 5 +- internal/appconfig/library_role_test.go | 5 +- internal/backuphost/runner.go | 75 ++++- internal/backuphost/runner_test.go | 37 +++ internal/governance/library_flow.go | 10 + internal/governance/library_operation.go | 2 +- internal/governance/library_verification.go | 51 ++++ .../governance/library_verification_test.go | 59 ++++ internal/httpapi/backup_schedule.go | 14 +- internal/httpapi/backup_schedule_test.go | 46 +++ internal/httpapi/library_flow.go | 9 +- internal/httpapi/library_restore.go | 12 +- internal/httpapi/library_verify.go | 219 ++++++++++++++ internal/httpapi/library_verify_test.go | 193 ++++++++++++ internal/httpapi/production_bindings.go | 54 ++++ internal/httpapi/production_bindings_test.go | 47 +++ internal/httpapi/server.go | 20 +- .../libraryverify}/canary_test.go | 2 +- internal/libraryverify/verify.go | 266 ++++++++++++++++ .../platform/postgres/library_dispatch.go | 3 +- .../platform/postgres/library_verification.go | 98 ++++++ ...0079_library_restore_verification.down.sql | 13 + ...000079_library_restore_verification.up.sql | 37 +++ internal/platform/postgres/migrations_test.go | 11 +- web/src/BackupScheduleSection.test.tsx | 26 ++ web/src/BackupScheduleSection.tsx | 21 +- web/src/ProductionBindingsSection.test.tsx | 101 +++++++ web/src/ProductionBindingsSection.tsx | 284 ++++++++++++------ web/src/ReleaseTargets.tsx | 10 +- web/src/RestoreVerificationSection.test.tsx | 103 +++++++ web/src/RestoreVerificationSection.tsx | 140 +++++++++ web/src/SystemSettings.test.tsx | 3 +- web/src/backupSchedule.ts | 3 + web/src/librarySnapshots.ts | 2 +- web/src/productionBindings.ts | 34 +++ web/src/restoreVerification.test.ts | 93 ++++++ web/src/restoreVerification.ts | 158 ++++++++++ 47 files changed, 2502 insertions(+), 353 deletions(-) delete mode 100644 cmd/librarydrill/canary.go create mode 100644 cmd/worker/library_verify.go create mode 100644 cmd/worker/library_verify_test.go create mode 100644 internal/governance/library_verification.go create mode 100644 internal/governance/library_verification_test.go create mode 100644 internal/httpapi/library_verify.go create mode 100644 internal/httpapi/library_verify_test.go rename {cmd/librarydrill => internal/libraryverify}/canary_test.go (99%) create mode 100644 internal/libraryverify/verify.go create mode 100644 internal/platform/postgres/library_verification.go create mode 100644 internal/platform/postgres/migrations/000079_library_restore_verification.down.sql create mode 100644 internal/platform/postgres/migrations/000079_library_restore_verification.up.sql create mode 100644 web/src/ProductionBindingsSection.test.tsx create mode 100644 web/src/RestoreVerificationSection.test.tsx create mode 100644 web/src/RestoreVerificationSection.tsx create mode 100644 web/src/restoreVerification.test.ts create mode 100644 web/src/restoreVerification.ts diff --git a/api/openapi.yaml b/api/openapi.yaml index f6cbedf..56bfad5 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1655,6 +1655,93 @@ } } }, + "/api/v1/library/verifications": { + "get": { + "tags": [ + "Runtime" + ], + "operationId": "getLibraryRestoreVerification", + "summary": "当前整库目标的恢复验证证据：verified 只由真实演练记录回答（按 target_id 绑定，换目标自动失效），并返回完整目标与失败原因", + "parameters": [ + { + "name": "scope", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "internal", + "external" + ] + } + } + ], + "responses": { + "200": { + "description": "目标与证据：verified/reason/latest/history/target" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "description": "恢复验证存储不可用或证据读取失败" + } + } + }, + "post": { + "tags": [ + "Runtime" + ], + "operationId": "startLibraryRestoreVerification", + "summary": "发起一次真实恢复验证（管理员）：维护隔离下捕获目标 → 校验基线 → 写 canary 证明校验有效 → 按预备份恢复 → 复核，通过后才产生同步闸口认的证据", + "responses": { + "202": { + "description": "已创建 verify 作业（worker 在维护窗口内真实执行）" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "403": { + "description": "不是管理员" + }, + "409": { + "description": "目标正在维护或已有在飞写作业（同步/恢复/基线/验证互斥）" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "description": "生产写入闸口关闭或恢复验证存储不可用" + } + }, + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": false, + "required": [ + "scope" + ], + "properties": { + "scope": { + "type": "string", + "enum": [ + "internal", + "external" + ] + } + } + } + } + } + } + } + }, "/api/v1/library/snapshots/{id}": { "get": { "tags": [ diff --git a/cmd/api/main.go b/cmd/api/main.go index 5a3b0a2..faac31e 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -232,6 +232,8 @@ func run(ctx context.Context, getenv func(string) string) error { LibraryProductionInternalKB: config.LibraryProductionInternalKB, LibraryProductionExternalKB: config.LibraryProductionExternalKB, LibraryProductionTenantID: config.LibraryProductionTenantID, + LibraryProductionIndex: config.LibraryProductionIndex, + LibraryProductionS3Bucket: config.LibraryProductionS3Bucket, LibraryRetentionVersions: config.LibraryRetentionVersions, LibraryRetentionDays: config.LibraryRetentionDays, BackupHost: backuphost.New(backuphost.Config{ diff --git a/cmd/librarydrill/canary.go b/cmd/librarydrill/canary.go deleted file mode 100644 index cb4a36c..0000000 --- a/cmd/librarydrill/canary.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "context" - "knowledge-governance/internal/libraryrelease" - v071 "knowledge-governance/internal/releaseadapter/v071" -) - -func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error { - ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false}) - if err := ledger.Seal(); err != nil { - return err - } - if _, err := v071.PersistWriteSet(root, ledger); err != nil { - return err - } - if err := mark(ctx); err != nil { - return err - } - return write(ctx) -} diff --git a/cmd/librarydrill/main.go b/cmd/librarydrill/main.go index c55b335..8af1685 100644 --- a/cmd/librarydrill/main.go +++ b/cmd/librarydrill/main.go @@ -1,4 +1,4 @@ -// command librarydrill 在目标实例上做一次**真实**的恢复演练： +// command librarydrill 是恢复演练的**命令行入口**，实现在 internal/libraryverify： // // 维护隔离（关入口/停写入者/证明排空） // → 捕获目标整库（真预备份，落归档） @@ -8,16 +8,14 @@ // → 逐表/索引/资源/Wiki 再校验一次，必须完全一致 // → 关窗口（按原运行状态恢复服务、开入口、验证健康） // -// 它只读/写**被测目标自己**，绝不触碰封存源；所有凭据来自进程环境 -// （与 worker 同一份 LIBRARY_* 配置），不进命令行、不打印。 -// 任一步失败都不伪装成功：窗口保持维护、报告写明失败阶段。 +// 为什么只剩薄封装：页面要能点一次"恢复验证"，worker 必须在进程内跑同一份实现。 +// 两份实现意味着"CLI 通过、页面失败"或反过来，演练结论就不可信了。 +// 它只读/写**被测目标自己**，绝不触碰封存源；所有凭据来自进程环境。 package main import ( "context" - "crypto/rand" "encoding/json" - "errors" "flag" "fmt" "log" @@ -26,35 +24,12 @@ import ( "time" "knowledge-governance/internal/appconfig" - "knowledge-governance/internal/libraryfreeze" - "knowledge-governance/internal/libraryrelease" - "knowledge-governance/internal/librarystack" + "knowledge-governance/internal/libraryverify" ) -type drillVerification struct { - SnapshotID string `json:"snapshot_id"` - Verified bool `json:"verified"` - Error string `json:"error,omitempty"` - At time.Time `json:"at"` -} - -type drillReport struct { - Status string `json:"status"` // passed | failed | maintenance_held - Scope string `json:"scope"` - Target string `json:"target"` - SnapshotID string `json:"snapshot_id"` - Project string `json:"project"` - Counts map[string]int64 `json:"counts"` - BaselineCheck *drillVerification `json:"baseline_check"` - DriftCheck *drillVerification `json:"drift_check"` // 必须失败：证明确实能发现差异 - RestoreCheck *drillVerification `json:"restore_check"` - Canary string `json:"canary"` - Failure string `json:"failure,omitempty"` - Stage string `json:"stage,omitempty"` - StartedAt time.Time `json:"started_at"` - FinishedAt time.Time `json:"finished_at"` - Until map[string]string `json:"until,omitempty"` -} +// 报告类型直接复用共享实现：命令行与页面/worker 的证据字段完全一致。 +type drillVerification = libraryverify.Verification +type drillReport = libraryverify.Report func main() { scope := flag.String("scope", "internal", "internal|external") @@ -88,18 +63,7 @@ func recoverWindows() error { if err != nil { return fmt.Errorf("load config: %w", err) } - manager, err := librarystack.OpenMaintenance(config) - if err != nil { - return err - } - defer manager.Close() - coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot) - if err != nil { - return err - } - defer control.Close() - defer coordinator.Close() - if err := coordinator.Recover(context.Background()); err != nil { + if err := libraryverify.RecoverWindows(config); err != nil { return err } log.Printf("maintenance windows recovered; target should be back to its original service state") @@ -107,170 +71,15 @@ func recoverWindows() error { } func run(scope, jobID string) (*drillReport, error) { - if scope != "internal" && scope != "external" { - return nil, errors.New("scope must be internal or external") - } config, err := appconfig.Load(os.Getenv) if err != nil { return nil, fmt.Errorf("load config: %w", err) } - ctx := context.Background() - report := &drillReport{Status: "failed", Scope: scope, Project: config.LibraryTargetProject, - Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()} - fail := func(stage string, cause error, held bool) (*drillReport, error) { - report.Stage = stage - report.Failure = cause.Error() - report.FinishedAt = time.Now().UTC() - if held { - report.Status = "maintenance_held" - } - return report, cause - } - - target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope) - if err != nil { - return fail("open_target", err, false) - } - defer target.Close() - manager, err := librarystack.OpenMaintenance(config) - if err != nil { - return fail("open_maintenance", err, false) - } - defer manager.Close() - if err := manager.VerifyTopology(ctx); err != nil { - return fail("topology", err, false) - } - coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot) - if err != nil { - return fail("open_coordinator", err, false) - } - defer control.Close() - defer coordinator.Close() - if jobID == "" { - jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405") + jobID = fmt.Sprintf("drill-%s-%d", scope, time.Now().UTC().UnixNano()) } - var ( - snapshotID string - counts map[string]int64 - canary string - contentTouched bool - baseline drillVerification - drift drillVerification - restored drillVerification - ) - work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) { - start := time.Now() - current, err := target.Current(ctx) - if err != nil { - return "", errors.New("读取目标当前整库状态失败：" + err.Error()) - } - counts = current.Counts - report.Counts = counts - // 1) 真预备份（落归档、内容寻址、导出后复核内容未变）。 - backup, err := backups.Prepare(ctx, "", current) - if err != nil { - return "", errors.New("捕获目标整库失败：" + err.Error()) - } - snapshotID = backup.SnapshotID - report.SnapshotID = snapshotID - if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil { - return "", errors.New("记录预备份身份失败：" + err.Error()) - } - log.Printf("drill: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start)) - - // 2) 基线校验：预备份必须与目标完全一致。 - baseline = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()} - if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil { - baseline.Error = err.Error() - report.BaselineCheck = &baseline - return "", errors.New("预备份与目标不一致（预备份本身不可信）：" + err.Error()) - } - baseline.Verified = true - report.BaselineCheck = &baseline - - // Canary is an isolated tag, including on an empty target. Never seed - // a document or change an existing row merely to make a drill possible. - var random [16]byte - if _, err := rand.Read(random[:]); err != nil { - return "", err - } - canary = fmt.Sprintf("%x", random[:]) - identityValue := target.Identity() - var exists bool - if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil { - return "", err - } - if exists { - return "", errors.New("canary identity collision; no write attempted") - } - ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope, - TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID, - BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID, - CreatedAt: time.Now().UTC(), Universe: backup.Universe} - report.Canary = canary - if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary, - func(ctx context.Context) error { - if err := window.MarkContentRisk(ctx); err != nil { - return err - } - contentTouched = true - return nil - }, func(ctx context.Context) error { - _, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary) - return err - }); err != nil { - return "", fmt.Errorf("canary preparation/write failed: %w", err) - } - drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()} - if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil { - drift.Error = "写入 canary 后校验仍然通过：校验没有真正比对内容" - report.DriftCheck = &drift - return "", errors.New("写入 canary 后校验仍然通过：校验没有真正比对内容，拒绝把这次演练当成通过") - } else { - drift.Error = verifyErr.Error() - } - report.DriftCheck = &drift - log.Printf("drill: drift detected as expected (%s)", drift.Error) - - // 4) 按预备份恢复。 - // Restore removes only the exact new canary identity in the durable write set. - if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID, - WriteSet: &ledger}); err != nil { - return "", errors.New("按预备份恢复失败：" + err.Error()) - } - // 5) 恢复后逐类复核：必须与预备份完全一致。 - restored = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()} - if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil { - restored.Error = err.Error() - report.RestoreCheck = &restored - return "", errors.New("恢复后内容仍与预备份不一致：" + err.Error()) - } - restored.Verified = true - report.RestoreCheck = &restored - log.Printf("drill: restored and verified snapshot=%s", snapshotID) - return snapshotID, nil - } - - window, windowErr := coordinator.RunWindow(ctx, jobID, - func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work, - // 没写过目标内容 ⇒ 按原状态恢复永远安全；写过 canary ⇒ 只有恢复并校验通过才开放。 - func(error) bool { return !contentTouched || restored.Verified }) - - report.Until = map[string]string{"job": jobID, "window_state": window.State} - if windowErr != nil || window.State != "succeeded" { - cause := windowErr - if cause == nil { - cause = fmt.Errorf("维护窗口状态 %s（未成功）", window.State) - } - if window.State == "held" { - return fail("window", cause, true) - } - return fail("window", cause, false) - } - report.Status = "passed" - report.FinishedAt = time.Now().UTC() - return report, nil + return libraryverify.Run(context.TODO(), config, scope, jobID, + func(stage, state, detail string) { log.Printf("stage=%s state=%s %s", stage, state, detail) }) } func writeReport(path string, report *drillReport) error { diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index bf4b17f..65393ef 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -233,8 +233,8 @@ func runLibraryReleaseAttempt(ctx context.Context, writer *libraryAttemptWriter, if !config.ProductionReleaseRequestsEnabled { gateReasons = append(gateReasons, "生产写入闸口关闭") } - if !config.ProductionBackupRestoreVerified { - gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据") + if ok, reason := libraryRestoreEvidenceSatisfied(ctx, store, config, payload.Scope); !ok { + gateReasons = append(gateReasons, reason) } if len(gateReasons) > 0 { message := "生产" + kindLabel(kind) + "被闸口拒绝：" + strings.Join(gateReasons, "；") @@ -628,6 +628,10 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore, // 基线作业被杀时窗口可能还停着目标：用同一个作业 id 接管那个窗口， // 继续捕获/校验，而不是留下一个 held 目标等人工处理。 queueType = libraryBaselineJobType + case governance.LibraryJobVerify: + // 恢复验证同样持有维护窗口：接管时必须用同一份作业 id 收掉那个窗口， + // 否则目标会一直停在维护里而作业显示失败。 + queueType = libraryVerifyJobType } payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope, "target_snapshot_id": job.TargetSnapshotID}) diff --git a/cmd/worker/library_release_handlers.go b/cmd/worker/library_release_handlers.go index 09a0586..cd9f247 100644 --- a/cmd/worker/library_release_handlers.go +++ b/cmd/worker/library_release_handlers.go @@ -30,4 +30,9 @@ func registerLibraryReleaseHandlers(handlers map[string]jobs.Handler, store *pos handlers[libraryBaselineJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { return runLibraryBaseline(ctx, store, config, job) }) + // 恢复验证：真实"备份→canary→恢复→复核"，通过后写入与当前目标绑定的可恢复证据。 + // 它是同步闸口的证据来源，所以与上面三类共用同一把执行锁与准入。 + handlers[libraryVerifyJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error { + return runLibraryVerify(ctx, store, config, job) + }) } diff --git a/cmd/worker/library_role_test.go b/cmd/worker/library_role_test.go index eb3efb7..16841bc 100644 --- a/cmd/worker/library_role_test.go +++ b/cmd/worker/library_role_test.go @@ -15,7 +15,8 @@ import ( func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) { handlers := map[string]jobs.Handler{} registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, nil) - want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType} + want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, + libraryBaselineJobType, libraryVerifyJobType} if len(handlers) != len(want) { t.Fatalf("registered %d handlers, want %d: %#v", len(handlers), len(want), handlers) } @@ -30,7 +31,8 @@ func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) { func TestRegisterLibraryReleaseHandlersFailsClosedWhenNotAdmitted(t *testing.T) { handlers := map[string]jobs.Handler{} registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, errNotAdmitted{}) - for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType} { + for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, + libraryBaselineJobType, libraryVerifyJobType} { err := handlers[jobType](context.Background(), jobs.Job{}) if err == nil || !strings.Contains(err.Error(), "准入未通过") { t.Fatalf("handler %q error = %v, want admission refusal", jobType, err) diff --git a/cmd/worker/library_verify.go b/cmd/worker/library_verify.go new file mode 100644 index 0000000..44e6015 --- /dev/null +++ b/cmd/worker/library_verify.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "time" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + "knowledge-governance/internal/libraryverify" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// libraryVerifyPayload 是"恢复验证"的队列载荷。 +// +// 它只带作业号与 scope：目标（库/索引/桶/实例）来自本进程的部署配置，不接受请求方 +// 指定。否则一次页面点击就能把"验证"引到别的库上，证据也就证明不了当前目标。 +type libraryVerifyPayload struct { + LibraryJobID string `json:"library_job_id"` + Scope string `json:"scope"` +} + +// runLibraryVerify 在目标实例上真实跑一次恢复验证，并把结果登记成证据。 +// +// 与 sync/restore/baseline 共用同一套准入与代次守卫：它会停写入者、在目标上写 canary +// 再按预备份恢复，所以必须走 admitLibraryJob + runOwnedLibraryAttempt，不能绕开。 +// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现，不允许两份。 +func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error { + var payload libraryVerifyPayload + if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || + (payload.Scope != "internal" && payload.Scope != "external") { + return errors.New("invalid library verify payload") + } + if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, + governance.LibraryJobVerify, true); admitErr != nil { + return admitErr + } + return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID, + func(runCtx context.Context, writer *libraryAttemptWriter) error { + return runLibraryVerifyAttempt(runCtx, writer, config, payload) + }) +} + +func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config, + payload libraryVerifyPayload) error { + store := writer + now := time.Now().UTC() + recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope, + operationID: writer.attempt.OperationID} + + existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID) + if jobErr != nil { + // 作业行读不到就没有可登记的终态：直接失败，不落任何证据。 + return fmt.Errorf("读不到业务作业 %s：%w", payload.LibraryJobID, jobErr) + } + if existing.State == governance.LibraryJobSucceeded { + // 幂等重放：已经成功过的验证不重复停目标、不重复写 canary。 + recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成，重复投递不重复执行") + return nil + } + if !config.ProductionReleaseRequestsEnabled { + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, + "生产写入闸口关闭（PRODUCTION_RELEASE_REQUESTS_ENABLED=false）：恢复验证要停写入者并在目标上写 canary，闸口关闭时不执行", + "capture") + } + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture", + "", "", "", "", now) + + report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID, + func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) }) + + // 证据只由真实执行产生：失败也照实登记 failure 与报告，绝不写成 passed。 + evidence := governance.LibraryRestoreVerification{ + ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID, + Scope: payload.Scope, + JobID: payload.LibraryJobID, + Status: governance.LibraryVerificationFailed, + StartedAt: now, + } + if report != nil { + evidence.StartedAt = report.StartedAt + evidence.SnapshotID = report.SnapshotID + evidence.Report = verificationReportObject(report) + } + if runErr == nil && report != nil && report.Status == "passed" { + evidence.Status = governance.LibraryVerificationPassed + evidence.FinishedAt = report.FinishedAt + } else { + evidence.FinishedAt = time.Now().UTC() + if report != nil && report.FinishedAt.After(evidence.StartedAt) { + evidence.FinishedAt = report.FinishedAt + } + switch { + case runErr != nil: + evidence.Failure = runErr.Error() + case report != nil && report.Status != "passed": + evidence.Failure = "恢复验证未通过（状态 " + report.Status + "）" + default: + evidence.Failure = "恢复验证未产生报告" + } + } + if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil { + // 证据落不了库就不能宣称验证通过：这会把闸口建立在一次没留痕的执行上。 + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, + "恢复验证执行结束但证据登记失败："+saveErr.Error(), "finalize") + } + + if runErr != nil || evidence.Status != governance.LibraryVerificationPassed { + held := report != nil && report.Status == "maintenance_held" + message := "恢复验证未通过：" + evidence.Failure + if held { + message += "（目标仍处于维护窗口，需要先按预备份恢复再重试）" + } + recovery := "未对目标执行任何写入，无需恢复" + if report != nil && report.SnapshotID != "" { + recovery = "目标已按预备份 " + report.SnapshotID + " 恢复并复核；下次验证会重新捕获" + } + snapshot := "" + if report != nil { + snapshot = report.SnapshotID + } + recorder.Stage(ctx, "finalize", "failed", message) + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", + message, recovery, snapshot, snapshot, time.Now().UTC()) + return errors.New(message) + } + + finishedAt := evidence.FinishedAt + if finishedAt.IsZero() { + finishedAt = time.Now().UTC() + } + recorder.Stage(ctx, "finalize", "succeeded", + "恢复验证通过：快照 "+evidence.SnapshotID+" 的预备份可真实恢复，证据已按当前目标登记") + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", + "", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt) + log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID) + return nil +} + +// verificationReportObject 把报告转成可落库的 JSON 对象（保留 canary/漂移校验等证据字段）。 +func verificationReportObject(report *libraryverify.Report) map[string]any { + if report == nil { + return nil + } + raw, err := json.Marshal(report) + if err != nil { + return map[string]any{"encode_error": err.Error()} + } + var object map[string]any + if err := json.Unmarshal(raw, &object); err != nil { + return map[string]any{"encode_error": err.Error()} + } + return object +} + +// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。 +// +// 证据是第一来源：library_restore_verifications 按当前 target_id 读取，换库/换索引/换桶 +// 之后旧证据自动失效，不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED +// 仍作为显式覆盖保留（不删除旧部署能力），但页面不再需要它，也不再由页面写入。 +func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface { + LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) +}, config appconfig.Config, scope string) (bool, string) { + if store != nil { + evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope) + switch { + case err != nil: + if config.ProductionBackupRestoreVerified { + return true, "" + } + return false, "读取恢复验证证据失败：" + err.Error() + case found: + if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok { + return true, "" + } else { + return false, reason + } + } + } + if config.ProductionBackupRestoreVerified { + return true, "" + } + return false, "当前目标还没有恢复验证证据：先跑一次恢复验证（备份 → canary → 恢复 → 复核）" +} diff --git a/cmd/worker/library_verify_test.go b/cmd/worker/library_verify_test.go new file mode 100644 index 0000000..2c35918 --- /dev/null +++ b/cmd/worker/library_verify_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" +) + +// 同步闸口的"恢复可验证"判断必须以当前目标的证据为准： +// 没有证据 ⇒ 拒绝并说清怎么补；有 passed 证据 ⇒ 放行；只有失败证据 ⇒ 拒绝并带失败原因。 +// 历史 env 覆盖只在证据缺失时兜底（页面不再写它）。 +type verifyEvidenceStore struct { + evidence governance.LibraryRestoreVerification + found bool +} + +func (s verifyEvidenceStore) LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) { + return s.evidence, s.found, nil +} + +func TestLibraryRestoreEvidenceSatisfied(t *testing.T) { + now := time.Now().UTC() + passed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{ + Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1", FinishedAt: now}} + if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), passed, appconfig.Config{}, "internal"); !ok { + t.Fatalf("passed evidence must satisfy gate: %q", reason) + } + failed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{ + Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-1", + Failure: "恢复后内容仍与预备份不一致", FinishedAt: now}} + if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), failed, appconfig.Config{}, "internal"); ok || + !strings.Contains(reason, "恢复后内容仍与预备份不一致") { + t.Fatalf("failed evidence must block with its own failure: ok=%v reason=%q", ok, reason) + } + missing := verifyEvidenceStore{} + if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing, appconfig.Config{}, "internal"); ok || + !strings.Contains(reason, "恢复验证") { + t.Fatalf("missing evidence must block and point at the remedy: ok=%v reason=%q", ok, reason) + } + // 历史部署的 env 覆盖：证据缺失时才生效，证据存在时不再起作用。 + if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing, + appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); !ok { + t.Fatalf("legacy env override must still allow when no evidence: %q", reason) + } + if ok, _ := libraryRestoreEvidenceSatisfied(context.Background(), failed, + appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); ok { + t.Fatal("evidence must win over the legacy env override") + } +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 55c33e6..eb86462 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -110,6 +110,10 @@ const libraryRestoreJobType = "library.restore" // libraryBaselineJobType 是"建立目标权威基线"的队列类型（与 API 侧常量同值）。 const libraryBaselineJobType = "library.baseline" +// libraryVerifyJobType 是"恢复验证"的队列类型：真实备份→canary→恢复→复核， +// 通过后才会产生同步闸口认的恢复证据。与 API 侧常量同值。 +const libraryVerifyJobType = "library.verify" + func main() { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() diff --git a/internal/appconfig/library_role.go b/internal/appconfig/library_role.go index c91fdfa..6e0a254 100644 --- a/internal/appconfig/library_role.go +++ b/internal/appconfig/library_role.go @@ -73,12 +73,13 @@ func (r LibraryWorkerRole) LibraryJobKinds() []string { switch r { case LibraryWorkerRoleReleaseExecutor: return []string{string(governance.LibraryJobSync), string(governance.LibraryJobRestore), - string(governance.LibraryJobRecover), string(governance.LibraryJobBaseline)} + string(governance.LibraryJobRecover), string(governance.LibraryJobBaseline), + string(governance.LibraryJobVerify)} case LibraryWorkerRoleCapture: return []string{string(governance.LibraryJobSeal)} default: return []string{string(governance.LibraryJobSeal), string(governance.LibraryJobSync), string(governance.LibraryJobRestore), string(governance.LibraryJobRecover), - string(governance.LibraryJobBaseline)} + string(governance.LibraryJobBaseline), string(governance.LibraryJobVerify)} } } diff --git a/internal/appconfig/library_role_test.go b/internal/appconfig/library_role_test.go index a92fc5b..ef64b15 100644 --- a/internal/appconfig/library_role_test.go +++ b/internal/appconfig/library_role_test.go @@ -63,13 +63,14 @@ func TestLoadRejectsUnknownLibraryWorkerRole(t *testing.T) { // 每个角色收拾哪些 kind、谁拥有 DDL，都必须互斥且覆盖完整。 func TestLibraryWorkerRoleOwnershipMaps(t *testing.T) { release := LibraryWorkerRoleReleaseExecutor - if got := release.LibraryJobKinds(); len(got) != 4 || got[0] != "sync" || got[1] != "restore" || got[2] != "recover" || got[3] != "baseline" { + if got := release.LibraryJobKinds(); len(got) != 5 || got[0] != "sync" || got[1] != "restore" || + got[2] != "recover" || got[3] != "baseline" || got[4] != "verify" { t.Fatalf("release-executor kinds = %v", got) } if got := LibraryWorkerRoleCapture.LibraryJobKinds(); len(got) != 1 || got[0] != "seal" { t.Fatalf("capture kinds = %v", got) } - if got := LibraryWorkerRoleFull.LibraryJobKinds(); len(got) != 5 { + if got := LibraryWorkerRoleFull.LibraryJobKinds(); len(got) != 6 { t.Fatalf("full kinds = %v", got) } for _, role := range []LibraryWorkerRole{LibraryWorkerRoleFull, LibraryWorkerRoleCapture} { diff --git a/internal/backuphost/runner.go b/internal/backuphost/runner.go index afe3a9d..46c71be 100644 --- a/internal/backuphost/runner.go +++ b/internal/backuphost/runner.go @@ -10,6 +10,8 @@ import ( "encoding/json" "errors" "fmt" + "net" + "os" "os/exec" "regexp" "strconv" @@ -127,12 +129,81 @@ func (r *Runner) runRemote(ctx context.Context, timeout time.Duration, remote st if ctx.Err() != nil { return text, fmt.Errorf("ssh %s 超时（%s）", r.config.Target, timeout) } - // 主机侧脚本会用退出码表达"参数非法/有库失败"，把输出带回去便于页面展示。 - return text, fmt.Errorf("ssh %s 执行失败: %s", r.config.Target, firstLines(text, 6)) + // 连接层失败与主机侧命令失败必须分开说：前者连脚本都没跑， + // 任何"脚本缺失/计划未下发"都是把未观测当成事实。 + return text, hostFailure(r.config.Target, text) } return text, nil } +// hostFailure 把 ssh 的非零退出分成两类：连不上，和连上了但远端命令失败。 +func hostFailure(target, text string) error { + if connectionFailurePattern.MatchString(text) { + return fmt.Errorf("目标主机 %s 不可达（%s）；未执行任何主机侧脚本，脚本与计划状态未知", + target, firstLines(text, 3)) + } + // 主机侧脚本会用退出码表达"参数非法/有库失败"，把输出带回去便于页面展示。 + return fmt.Errorf("已连上 %s，但主机侧命令失败: %s", target, firstLines(text, 6)) +} + +// connectionFailurePattern 是 OpenSSH 客户端在"根本没建立连接"时打的原文。 +// 只有命中它才允许说"不可达"；其余非零退出属于远端命令自己的失败。 +var connectionFailurePattern = regexp.MustCompile(`(?i)ssh: connect to host|Operation timed out|Connection timed out|Connection refused|Connection reset by peer|No route to host|Network is unreachable|Could not resolve hostname|Host key verification failed|Permission denied|Connection closed by`) + +// Endpoint 读回挂载的 ssh 配置里本别名真实拨号的 host:port。 +// 页面必须能说清"现在连的是哪个地址"——只显示别名时，改错文件也看不出来。 +func (r *Runner) Endpoint() string { + if r == nil || r.config.SSHConfig == "" || r.config.Target == "" { + return "" + } + raw, err := os.ReadFile(r.config.SSHConfig) + if err != nil { + return "" + } + host, port := sshConfigEndpoint(string(raw), r.config.Target) + if host == "" { + return "" + } + if port == "" { + port = "22" + } + return net.JoinHostPort(host, port) +} + +// sshConfigEndpoint 解析 `Host ` 段的 HostName/Port（首个匹配段生效）。 +// 只做"页面显示用"的最小解析，不替代 ssh 自己的配置语义。 +func sshConfigEndpoint(config, alias string) (string, string) { + host, port := "", "" + active := false + for _, line := range strings.Split(config, "\n") { + fields := strings.Fields(strings.TrimSpace(line)) + if len(fields) == 0 || strings.HasPrefix(fields[0], "#") { + continue + } + key := strings.ToLower(fields[0]) + if key == "host" { + active = false + for _, pattern := range fields[1:] { + if pattern == alias { + active = true + break + } + } + continue + } + if !active || len(fields) < 2 { + continue + } + switch key { + case "hostname": + host = strings.Trim(fields[1], `"`) + case "port": + port = strings.Trim(fields[1], `"`) + } + } + return host, port +} + var scheduleAppliedPattern = regexp.MustCompile( `SCHEDULE_APPLIED enabled=(\d)(?: time=(\S+))?(?: days=(\d+))?(?: months=(\d+))?(?: next=(\S+))?(?: cron=(\S+))?`) diff --git a/internal/backuphost/runner_test.go b/internal/backuphost/runner_test.go index 5794158..3e538d9 100644 --- a/internal/backuphost/runner_test.go +++ b/internal/backuphost/runner_test.go @@ -3,6 +3,8 @@ package backuphost import ( "context" "errors" + "os" + "path/filepath" "strings" "testing" "time" @@ -188,6 +190,41 @@ func TestRunnerSurfacesHostFailure(t *testing.T) { } } +// 现场原错：ssh: connect to host 120.76.217.51 port 22: Operation timed out。 +// 连不上时脚本根本没跑，只能说"不可达"；断言配对的反面：不能说脚本缺失。 +func TestHostFailureSeparatesUnreachableFromRemoteCommand(t *testing.T) { + unreachable := hostFailure("weknora-test", "ssh: connect to host 120.76.217.51 port 22: Operation timed out") + if !strings.Contains(unreachable.Error(), "不可达") || !strings.Contains(unreachable.Error(), "Operation timed out") { + t.Fatalf("连接超时应报不可达并带原错: %v", unreachable) + } + if strings.Contains(unreachable.Error(), "主机侧命令失败") || strings.Contains(unreachable.Error(), "脚本缺失") { + t.Fatalf("连接失败不能报成远端命令/脚本问题: %v", unreachable) + } + remote := hostFailure("weknora-test", "bash: /opt/weknora-test/logical-backup.sh: No such file or directory") + if !strings.Contains(remote.Error(), "已连上") || !strings.Contains(remote.Error(), "No such file") { + t.Fatalf("已连接但脚本失败时应原样带回原因: %v", remote) + } + if strings.Contains(remote.Error(), "不可达") { + t.Fatalf("已连上就不能说不可达: %v", remote) + } +} + +// 页面要能说清"实际拨号的地址"，否则改错 ssh_config 也看不出来。 +func TestRunnerEndpointReadsDialedHostPort(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssh_config") + config := "Host other\n HostName 10.0.0.9\n Port 2222\n\nHost weknora-test\n HostName 100.85.111.121\n HostKeyAlias 120.76.217.51\n" + if err := os.WriteFile(path, []byte(config), 0o600); err != nil { + t.Fatal(err) + } + runner := New(Config{SSHConfig: path, Target: "weknora-test", ScriptDir: "/opt/weknora-test"}) + if got := runner.Endpoint(); got != "100.85.111.121:22" { + t.Fatalf("应报真实拨号地址（默认 22），得到 %q", got) + } + if got := New(Config{SSHConfig: filepath.Join(t.TempDir(), "missing"), Target: "weknora-test"}).Endpoint(); got != "" { + t.Fatalf("配置文件不可读时应返回空，不猜地址，得到 %q", got) + } +} + func TestRunnerRunNowReadsBackLastRun(t *testing.T) { calls := 0 runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) { diff --git a/internal/governance/library_flow.go b/internal/governance/library_flow.go index 0a08b5b..3773f6f 100644 --- a/internal/governance/library_flow.go +++ b/internal/governance/library_flow.go @@ -44,6 +44,10 @@ const ( // 读出来封存成快照并逐类校验，再登记成 production_effective。 // 它不写目标内容、不清空目标、不猜版本——基线只能来自真实捕获与校验。 LibraryJobBaseline LibraryJobKind = "baseline" + // LibraryJobVerify 是"恢复验证"：在维护窗口内捕获目标 → 校验基线一致 → + // 写一个 canary 证明校验能发现差异 → 按预备份恢复 → 复核完全一致。 + // 它是唯一能产生"生产可恢复"证据的作业，页面不再接受手填开关。 + LibraryJobVerify LibraryJobKind = "verify" ) type LibraryJobState string @@ -173,6 +177,12 @@ func BaselineStages() []string { return []string{"capture", "verify", "finalize"} } +// VerifyStages 是恢复验证的阶段：捕获 → 基线校验 → 写 canary（证明校验有效） +// → 按预备份恢复 → 恢复后复核。canary 与 restore 都是真实写入，失败必须可见。 +func VerifyStages() []string { + return []string{"capture", "verify", "canary", "restore", "finalize"} +} + // LibraryReleaseAuthorization 是生产发布授权：绑定目标、快照、参数摘要与有效期。 type LibraryReleaseAuthorization struct { ID string `json:"id"` diff --git a/internal/governance/library_operation.go b/internal/governance/library_operation.go index ac9c346..f258db1 100644 --- a/internal/governance/library_operation.go +++ b/internal/governance/library_operation.go @@ -147,7 +147,7 @@ func ValidateLibraryOperation(op LibraryOperation) error { return errors.New("library operation requires internal or external scope") } switch op.Kind { - case LibraryJobSeal, LibraryJobSync, LibraryJobRestore, LibraryJobBaseline, LibraryJobRecover: + case LibraryJobSeal, LibraryJobSync, LibraryJobRestore, LibraryJobBaseline, LibraryJobRecover, LibraryJobVerify: default: return fmt.Errorf("unsupported library operation kind %q", op.Kind) } diff --git a/internal/governance/library_verification.go b/internal/governance/library_verification.go new file mode 100644 index 0000000..306285b --- /dev/null +++ b/internal/governance/library_verification.go @@ -0,0 +1,51 @@ +package governance + +import ( + "strings" + "time" +) + +// LibraryRestoreVerification 是一次**真实**恢复验证（备份 → canary → 恢复 → 复核）的登记。 +// +// 为什么不能继续用环境变量 PRODUCTION_BACKUP_RESTORE_VERIFIED：那是一个可以手填的开关， +// 它既证明不了"有人真跑过恢复"，也证明不了"跑的是当前目标"。这里的每条记录由 worker +// 在真实演练结束后写入，绑定 target_id（实例+库+索引+桶）与 scope，失败也照实留痕。 +type LibraryRestoreVerification struct { + ID string `json:"id"` + Scope string `json:"scope"` + Status string `json:"status"` // passed | failed + SnapshotID string `json:"snapshot_id"` + JobID string `json:"job_id"` + Failure string `json:"failure,omitempty"` + Report map[string]any `json:"report,omitempty"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` +} + +const ( + LibraryVerificationPassed = "passed" + LibraryVerificationFailed = "failed" +) + +// LibraryVerificationSatisfied 只回答一件事：这条证据能不能支撑"当前目标的恢复验证已通过"。 +// 通过的条件必须是全部成立——状态 passed、scope 相同、快照非空、完成时间有效。 +// target_id 的匹配由查询层（按当前 target_id 过滤）保证，这里不重复比较。 +func LibraryVerificationSatisfied(verification LibraryRestoreVerification, scope string) (bool, string) { + if verification.Status != LibraryVerificationPassed { + reason := "最近一次恢复验证没有通过" + if detail := strings.TrimSpace(verification.Failure); detail != "" { + reason += "：" + detail + } + return false, reason + } + if verification.Scope != scope { + return false, "恢复验证证据属于另一个 scope（" + verification.Scope + "）" + } + if strings.TrimSpace(verification.SnapshotID) == "" { + return false, "恢复验证证据没有绑定快照，不能作为备份可恢复的证明" + } + if verification.FinishedAt.IsZero() { + return false, "恢复验证证据没有完成时间" + } + return true, "" +} diff --git a/internal/governance/library_verification_test.go b/internal/governance/library_verification_test.go new file mode 100644 index 0000000..97e2709 --- /dev/null +++ b/internal/governance/library_verification_test.go @@ -0,0 +1,59 @@ +package governance + +import ( + "testing" + "time" +) + +// 证据能不能支撑"当前目标可恢复"只由这四个条件回答： +// 状态 passed、scope 相同、绑定了快照、有完成时间。少任何一条都不算。 +func TestLibraryVerificationSatisfied(t *testing.T) { + now := time.Now().UTC() + cases := []struct { + name string + verification LibraryRestoreVerification + scope string + want bool + reason string + }{ + {name: "passed with snapshot", scope: "internal", want: true, + verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal", + SnapshotID: "snap-1", FinishedAt: now}}, + {name: "failed carries failure", scope: "internal", want: false, reason: "canary 后校验仍通过", + verification: LibraryRestoreVerification{Status: LibraryVerificationFailed, Scope: "internal", + SnapshotID: "snap-1", Failure: "canary 后校验仍通过", FinishedAt: now}}, + {name: "wrong scope", scope: "internal", want: false, reason: "另一个 scope", + verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "external", + SnapshotID: "snap-1", FinishedAt: now}}, + {name: "no snapshot", scope: "internal", want: false, reason: "没有绑定快照", + verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal", + FinishedAt: now}}, + {name: "no finish time", scope: "internal", want: false, reason: "没有完成时间", + verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal", + SnapshotID: "snap-1"}}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + ok, reason := LibraryVerificationSatisfied(testCase.verification, testCase.scope) + if ok != testCase.want { + t.Fatalf("satisfied=%v want=%v reason=%q", ok, testCase.want, reason) + } + if !testCase.want && testCase.reason != "" && !containsText(reason, testCase.reason) { + t.Fatalf("reason=%q want contains %q", reason, testCase.reason) + } + }) + } +} + +func containsText(haystack, needle string) bool { + return len(haystack) >= len(needle) && (haystack == needle || indexOf(haystack, needle) >= 0) +} + +func indexOf(haystack, needle string) int { + for i := 0; i+len(needle) <= len(haystack); i++ { + if haystack[i:i+len(needle)] == needle { + return i + } + } + return -1 +} diff --git a/internal/httpapi/backup_schedule.go b/internal/httpapi/backup_schedule.go index 72e9572..92e14fe 100644 --- a/internal/httpapi/backup_schedule.go +++ b/internal/httpapi/backup_schedule.go @@ -24,6 +24,9 @@ type BackupScheduleStore interface { type BackupHost interface { Available() bool Target() string + // Endpoint 是挂载的 ssh 配置里该别名真实拨号的 host:port；页面据此证明 + // "连的是哪条通道"，而不是只显示一个别名。 + Endpoint() string Show(context.Context) (backuphost.ShowResult, error) Apply(context.Context, bool, string, int, int) (backuphost.ApplyResult, error) RunNow(context.Context) (backuphost.RunResult, error) @@ -82,8 +85,11 @@ type backupScheduleView struct { // HostConfigured/HostReachable 区分"没接线"与"接线了但连不上"。 HostConfigured bool `json:"host_configured"` HostTarget string `json:"host_target"` - HostReachable bool `json:"host_reachable"` - HostError string `json:"host_error"` + // HostEndpoint 是这条别名真实拨号的 host:port（来自挂载的 ssh 配置）。 + // 连接失败时它就是"到底往哪连"的唯一证据，不能只给一个别名让人猜。 + HostEndpoint string `json:"host_endpoint"` + HostReachable bool `json:"host_reachable"` + HostError string `json:"host_error"` HostScheduleEnabled bool `json:"host_schedule_enabled"` HostScheduleTime string `json:"host_schedule_time"` @@ -281,6 +287,7 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool) } view.HostConfigured = config.BackupHost.Available() view.HostTarget = config.BackupHost.Target() + view.HostEndpoint = config.BackupHost.Endpoint() if !view.HostConfigured { view.InSync = false view.HostError = "目标主机凭据未挂载（BACKUP_HOST_SSH_CONFIG / BACKUP_HOST_TARGET）" @@ -293,6 +300,9 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool) if err != nil { view.HostReachable = false view.HostError = err.Error() + if view.HostEndpoint != "" { + view.HostError += "（拨号地址 " + view.HostEndpoint + "）" + } view.InSync = false } else { view.HostReachable = true diff --git a/internal/httpapi/backup_schedule_test.go b/internal/httpapi/backup_schedule_test.go index 01e4e79..8d08bef 100644 --- a/internal/httpapi/backup_schedule_test.go +++ b/internal/httpapi/backup_schedule_test.go @@ -46,6 +46,7 @@ func (store *fakeBackupScheduleStore) MarkBackupScheduleApplied(_ context.Contex type fakeBackupHost struct { available bool target string + endpoint string show backuphost.ShowResult showErr error apply backuphost.ApplyResult @@ -59,6 +60,9 @@ type fakeBackupHost struct { func (host *fakeBackupHost) Available() bool { return host.available } func (host *fakeBackupHost) Target() string { return host.target } +func (host *fakeBackupHost) Endpoint() string { + return host.endpoint +} func (host *fakeBackupHost) Show(context.Context) (backuphost.ShowResult, error) { return host.show, host.showErr } @@ -166,6 +170,48 @@ func TestHandleBackupScheduleHostFlagsDrift(t *testing.T) { } } +func TestHandleBackupScheduleHostUnreachableSeparatesConnectionFromHostState(t *testing.T) { + // 现场原错：ssh connect to host 120.76.217.51 port 22 Operation timed out。 + // 连接没建立 ⇒ 既不能报"脚本缺失"，也不能报"计划未下发"，只能报不可达 + 拨号地址。 + host := &fakeBackupHost{ + available: true, target: "weknora-test", endpoint: "100.85.111.121:22", + showErr: errors.New("目标主机 weknora-test 不可达（ssh: connect to host 120.76.217.51 port 22: Operation timed out）；未执行任何主机侧脚本，脚本与计划状态未知"), + } + config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host) + recorder := httptest.NewRecorder() + config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil)) + data := decodeData(t, recorder) + if data["host_reachable"] != false || data["in_sync"] != false { + t.Fatalf("连不上时不能报告可达或已同步: %+v", data) + } + message := data["host_error"].(string) + if !strings.Contains(message, "不可达") || !strings.Contains(message, "100.85.111.121:22") { + t.Fatalf("应报不可达并给出真实拨号地址: %q", message) + } + if strings.Contains(message, "脚本缺失") || strings.Contains(message, "未下发") { + t.Fatalf("未观测到的脚本/计划状态不能当成结论: %q", message) + } +} + +func TestHandleBackupScheduleHostReportsMissingScriptOnlyAfterConnecting(t *testing.T) { + // 真正的脚本缺失：ssh 已连上，主机回的是脚本不存在 ⇒ 这时才允许说脚本缺失。 + host := &fakeBackupHost{ + available: true, target: "weknora-test", endpoint: "100.85.111.121:22", + showErr: errors.New("已连上 weknora-test，但主机侧命令失败: bash: /opt/weknora-test/install_backup_schedule.sh: No such file or directory"), + } + config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host) + recorder := httptest.NewRecorder() + config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil)) + data := decodeData(t, recorder) + message := data["host_error"].(string) + if !strings.Contains(message, "已连上") || !strings.Contains(message, "No such file") { + t.Fatalf("已连接但脚本缺失时应原样带回主机原因: %q", message) + } + if strings.Contains(message, "不可达") { + t.Fatalf("已连上就不能说不可达: %q", message) + } +} + func TestHandleBackupScheduleHostUnconfiguredDegrades(t *testing.T) { config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, &fakeBackupHost{available: false}) recorder := httptest.NewRecorder() diff --git a/internal/httpapi/library_flow.go b/internal/httpapi/library_flow.go index 18cac27..f1bd640 100644 --- a/internal/httpapi/library_flow.go +++ b/internal/httpapi/library_flow.go @@ -642,9 +642,7 @@ func librarySyncGateWithGrant(config Config, production libraryKnownView, prebac if !config.ProductionReleaseRequestsEnabled { gate.Reasons = append(gate.Reasons, "生产写入闸口关闭（PRODUCTION_RELEASE_REQUESTS_ENABLED=false）") } - if !config.ProductionBackupRestoreVerified { - gate.Reasons = append(gate.Reasons, "没有已核验的生产恢复演练证据（PRODUCTION_BACKUP_RESTORE_VERIFIED=false）") - } + gate.Reasons = append(gate.Reasons, restoreGateReasons(config)...) // 生产端点以"整库目标是否配置"为准：整库同步直接写 PG/ES/对象， // 不依赖 WeKnora 的 HTTP 端，所以不能拿它是否配置当门禁。 if strings.TrimSpace(config.LibraryProductionDatabaseURL) == "" { @@ -814,7 +812,8 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi grant, hasGrant := activeGrantFor(r.Context(), config, scope, target) overview.Authorization = authorizationViewOf(grant, hasGrant) - overview.Gate = librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, target, hasGrant) + // 闸口按当前目标的真实恢复证据判断（没有证据时也接受历史 env 覆盖）。 + overview.Gate = librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), scope), overview.Production, overview.Prebackup, target, hasGrant) writeJSON(w, http.StatusOK, overview) } @@ -963,7 +962,7 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { return } } - gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant) + gate := librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), request.Scope), overview.Production, overview.Prebackup, request.Target, hasGrant) if !gate.SyncAllowed { writeJSON(w, http.StatusServiceUnavailable, map[string]any{ "code": "library_sync_blocked", "reasons": gate.Reasons, diff --git a/internal/httpapi/library_restore.go b/internal/httpapi/library_restore.go index a30e822..3b40884 100644 --- a/internal/httpapi/library_restore.go +++ b/internal/httpapi/library_restore.go @@ -71,9 +71,7 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config if !config.ProductionReleaseRequestsEnabled { reasons = append(reasons, "生产写入闸口关闭") } - if !config.ProductionBackupRestoreVerified { - reasons = append(reasons, "没有已核验的生产恢复演练证据") - } + reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), request.Scope))...) authorizationOK := false if storeWithAuth, ok := config.Store.(interface { ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error) @@ -169,9 +167,7 @@ func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config C if !config.ProductionReleaseRequestsEnabled { view.Reasons = append(view.Reasons, "生产写入闸口关闭（PRODUCTION_RELEASE_REQUESTS_ENABLED=false）") } - if !config.ProductionBackupRestoreVerified { - view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据（PRODUCTION_BACKUP_RESTORE_VERIFIED=false）") - } + view.Reasons = append(view.Reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), scope))...) digest, err := libraryRestoreParamsDigest(scope, snapshotID) if err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"}) @@ -228,9 +224,7 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) { if !config.ProductionReleaseRequestsEnabled { reasons = append(reasons, "生产写入闸口关闭") } - if !config.ProductionBackupRestoreVerified { - reasons = append(reasons, "没有已核验的生产恢复演练证据") - } + reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), source.Scope))...) var digest string switch source.Kind { case governance.LibraryJobSync: diff --git a/internal/httpapi/library_verify.go b/internal/httpapi/library_verify.go new file mode 100644 index 0000000..4ce1bc8 --- /dev/null +++ b/internal/httpapi/library_verify.go @@ -0,0 +1,219 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "knowledge-governance/internal/governance" +) + +// 恢复验证：页面点一次"跑恢复验证"，后端建一个持久作业，worker 用与 CLI 同一份实现 +// （internal/libraryverify）真实执行：捕获目标 → 基线校验 → 写 canary 证明校验有效 → +// 按预备份恢复 → 恢复后复核。通过与否只由这条证据表回答，页面不接受手填开关。 +const libraryVerifyJobType = "library.verify" + +type libraryVerifyRequest struct { + Scope string `json:"scope"` +} + +// libraryVerificationView 是页面需要的全部事实：当前目标是谁、最近一次验证结果、 +// 证据是否对当前目标成立、以及同步闸口能不能据此放行。 +type libraryVerificationView struct { + Scope string `json:"scope"` + Target libraryVerificationTarget `json:"target"` + Verified bool `json:"verified"` + Reason string `json:"reason"` + Latest *governance.LibraryRestoreVerification `json:"latest,omitempty"` + History []governance.LibraryRestoreVerification `json:"history"` + Running bool `json:"running"` + JobID string `json:"job_id,omitempty"` +} + +type libraryVerificationTarget struct { + Instance string `json:"instance"` + Project string `json:"project"` + InternalKB string `json:"internal_kb"` + ExternalKB string `json:"external_kb"` + Index string `json:"index"` + S3Bucket string `json:"s3_bucket"` + TenantID uint64 `json:"tenant_id"` + DatabaseSet bool `json:"database_configured"` +} + +// libraryVerifyEnqueue 投递一次恢复验证作业（管理员）。 +// +// 它不签发发布授权、不改版本：验证只回答"当前目标的预备份能不能恢复"。 +// 目标是部署配置，不接受请求体指定——否则一次点击就能验证别的库并把证据挂到当前目标上。 +func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config) { + if r.Method != http.MethodPost { + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"}) + return + } + identity, ok := sessionFrom(r.Context()) + if !ok || roleRank(identity.Role) < roleRank(roleAdmin) { + writeJSON(w, http.StatusForbidden, map[string]any{"error": "恢复验证仅管理员可发起"}) + return + } + var request libraryVerifyRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + if request.Scope != "internal" && request.Scope != "external" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"}) + return + } + store, ok := libraryStore(config) + if !ok { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_verify_unavailable", + "error": "恢复验证存储不可用"}) + return + } + // 验证要停写入者并在目标上写 canary：维护中或闸口关闭时直接拒绝， + // 不让页面投一个 worker 必然失败的作业。 + if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held { + writeJSON(w, http.StatusConflict, body) + return + } + if !config.ProductionReleaseRequestsEnabled { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{ + "code": "library_verify_blocked", + "reasons": []string{ + "生产写入闸口关闭（PRODUCTION_RELEASE_REQUESTS_ENABLED=false）：恢复验证要停写入者并在目标上写 canary", + }, + }) + return + } + digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobVerify, + map[string]any{"scope": request.Scope}) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"}) + return + } + now := time.Now().UTC() + job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{ + ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope, + Kind: governance.LibraryJobVerify, + IdempotencyKey: "library.verify:" + request.Scope + ":" + now.Format("20060102T150405.000000000"), + RequestDigest: digest, Actor: identity.Username, + }, governance.VerifyStages(), false, now) + if errors.Is(err, governance.ErrLibraryJobActive) { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active", + "error": "该目标已有在飞的写作业（同步/恢复/基线/验证互斥），等它结束再发起"}) + return + } + if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict", + "error": "同一幂等键对应了不同的请求"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify job"}) + return + } + writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": reused}) +} + +// libraryVerifications 读当前目标的恢复验证证据（页面轮询用）。 +func libraryVerifications(w http.ResponseWriter, r *http.Request, config Config) { + scope, err := scopeFromQuery(r) + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + view := libraryVerificationView{ + Scope: scope, + History: []governance.LibraryRestoreVerification{}, + Target: libraryVerificationTarget{ + Instance: config.LibraryTargetInstance, + Project: config.LibraryTargetProject, + InternalKB: config.LibraryProductionInternalKB, + ExternalKB: config.LibraryProductionExternalKB, + Index: config.LibraryProductionIndex, + S3Bucket: config.LibraryProductionS3Bucket, + TenantID: config.LibraryProductionTenantID, + DatabaseSet: config.LibraryProductionDatabaseURL != "", + }, + } + store, ok := config.Store.(interface { + LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) + LibraryRestoreVerifications(context.Context, string, int) ([]governance.LibraryRestoreVerification, error) + }) + if !ok { + view.Reason = "当前部署没有恢复验证存储：无法读取证据" + writeJSON(w, http.StatusServiceUnavailable, view) + return + } + latest, found, err := store.LatestLibraryRestoreVerification(r.Context(), scope) + if err != nil { + view.Reason = "读取恢复验证证据失败：" + err.Error() + writeJSON(w, http.StatusServiceUnavailable, view) + return + } + if found { + view.Latest = &latest + view.JobID = latest.JobID + } + if history, err := store.LibraryRestoreVerifications(r.Context(), scope, 10); err == nil { + view.History = history + } + view.Verified, view.Reason = governance.LibraryVerificationSatisfied(latest, scope) + if !found { + view.Verified = false + view.Reason = "还没有对当前目标做过恢复验证：点「跑一次恢复验证」（真实备份 → canary → 恢复 → 复核）" + } + writeJSON(w, http.StatusOK, view) +} + +// withRestoreEvidence 把"当前目标是否有真实恢复证据"注入 Config 的副本， +// 让同步/恢复闸口在同一个请求里按证据判断，而不是读一个可能被手填的 env。 +// 读不到证据时不改写任何东西：闸口照旧因为缺证据而拒绝。 +func (c Config) withRestoreEvidence(ctx context.Context, scope string) Config { + if ok, err := c.restoreEvidence(ctx, scope); err == nil { + c.restoreVerified = ok + c.restoreEvidenceReason = "" + } else { + c.restoreVerified = false + c.restoreEvidenceReason = err.Error() + } + return c +} + +// restoreEvidence 读当前目标的最近一条证据并给出它能不能支撑"可恢复"。 +func (c Config) restoreEvidence(ctx context.Context, scope string) (bool, error) { + store, ok := c.Store.(interface { + LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) + }) + if !ok { + return false, errors.New("没有已核验的生产恢复演练证据") + } + latest, found, err := store.LatestLibraryRestoreVerification(ctx, scope) + if err != nil { + return false, fmt.Errorf("读取恢复验证证据失败：%w", err) + } + if !found { + return false, errors.New("当前目标还没有恢复验证证据：先在「生产保护与恢复」里跑一次恢复验证") + } + ok, reason := governance.LibraryVerificationSatisfied(latest, scope) + if !ok { + return false, errors.New(reason) + } + return true, nil +} + +// restoreGateReasons 是闸口对"恢复可验证"的统一判断：证据优先， +// 历史 env 覆盖仅在证据缺失时兜底（页面不再写入它）。 +func restoreGateReasons(config Config) []string { + if config.restoreVerified || config.ProductionBackupRestoreVerified { + return nil + } + reason := config.restoreEvidenceReason + if reason == "" { + reason = "没有已核验的生产恢复演练证据" + } + return []string{reason} +} diff --git a/internal/httpapi/library_verify_test.go b/internal/httpapi/library_verify_test.go new file mode 100644 index 0000000..7fd2670 --- /dev/null +++ b/internal/httpapi/library_verify_test.go @@ -0,0 +1,193 @@ +package httpapi + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "knowledge-governance/internal/governance" +) + +// 恢复验证的页面契约： +// - 只有管理员能发起（它会停写入者并在目标上写 canary）； +// - 闸口关闭 / 维护中时直接拒绝，不投一个 worker 必然失败的作业； +// - 证据只由真实执行写入：没有证据 ⇒ verified=false，过期/换目标 ⇒ 失效。 +// +// 这里不模拟"真跑一次恢复"（那需要真实目标）；执行体本身在 internal/libraryverify， +// 由库内 CLI 与 worker 共用，属于隔离 fixture 之外的集成范围。 + +type verificationStore struct { + fakeLibraryStore + verifications []governance.LibraryRestoreVerification +} + +func (s *verificationStore) LatestLibraryRestoreVerification(_ context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) { + for _, item := range s.verifications { + if item.Scope == scope { + return item, true, nil + } + } + return governance.LibraryRestoreVerification{}, false, nil +} + +func (s *verificationStore) LibraryRestoreVerifications(_ context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) { + items := make([]governance.LibraryRestoreVerification, 0, len(s.verifications)) + for _, item := range s.verifications { + if item.Scope == scope { + items = append(items, item) + } + } + if limit > 0 && len(items) > limit { + items = items[:limit] + } + return items, nil +} + +func verifyRequest(body, role, username string) *http.Request { + request := httptest.NewRequest(http.MethodPost, "/api/v1/library/verifications", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + ctx := context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: username, Role: role}) + return request.WithContext(ctx) +} + +func TestLibraryVerifyRequiresAdmin(t *testing.T) { + store := &verificationStore{} + recorder := httptest.NewRecorder() + libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleEngineer, "eng"), + Config{Store: store, ProductionReleaseRequestsEnabled: true}) + if recorder.Code != http.StatusForbidden { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if store.createN != 0 { + t.Fatal("non-admin must not create a verify job") + } +} + +func TestLibraryVerifyRefusesWhenWriteGateClosed(t *testing.T) { + store := &verificationStore{} + recorder := httptest.NewRecorder() + libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"), + Config{Store: store, ProductionReleaseRequestsEnabled: false}) + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + body := decodeBody(t, recorder) + if body["code"] != "library_verify_blocked" { + t.Fatalf("body=%v", body) + } + if store.createN != 0 { + t.Fatal("closed gate must not create a verify job") + } +} + +func TestLibraryVerifyEnqueuesVerifyJobWithCanaryStages(t *testing.T) { + store := &verificationStore{} + store.enqueueEchoRequestedID = true + recorder := httptest.NewRecorder() + libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"), + Config{Store: store, ProductionReleaseRequestsEnabled: true}) + if recorder.Code != http.StatusAccepted { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if store.lastJob.Kind != governance.LibraryJobVerify || store.lastJob.Scope != "internal" { + t.Fatalf("job=%#v", store.lastJob) + } + if !strings.HasPrefix(store.lastJob.ID, "libverify-internal-") { + t.Fatalf("job id=%q", store.lastJob.ID) + } + // 阶段必须包含 canary：没有"写 canary 证明校验有效"这一步，验证就只是自证。 + want := governance.VerifyStages() + if len(want) != 5 || want[2] != "canary" || want[3] != "restore" || want[4] != "finalize" { + t.Fatalf("verify stages=%v", want) + } +} + +func TestLibraryVerificationsWithoutEvidenceIsNotVerified(t *testing.T) { + store := &verificationStore{} + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil) + libraryVerifications(recorder, request, Config{Store: store}) + if recorder.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + body := decodeBody(t, recorder) + if body["verified"] != false { + t.Fatalf("verified must be false without evidence: %v", body) + } + if reason, _ := body["reason"].(string); !strings.Contains(reason, "还没有对当前目标做过恢复验证") { + t.Fatalf("reason=%q", reason) + } +} + +func TestLibraryVerificationsWithPassedEvidenceIsVerified(t *testing.T) { + store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{ + ID: "libverify-internal-job1", Scope: "internal", Status: governance.LibraryVerificationPassed, + SnapshotID: "snap-1", JobID: "job1", StartedAt: time.Now().UTC().Add(-time.Minute), + FinishedAt: time.Now().UTC(), + }}} + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil) + libraryVerifications(recorder, request, Config{Store: store}) + body := decodeBody(t, recorder) + if body["verified"] != true { + t.Fatalf("verified must be true with passed evidence: %v", body) + } +} + +func TestLibraryVerificationsFailedEvidenceIsNotVerified(t *testing.T) { + store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{ + ID: "libverify-internal-job2", Scope: "internal", Status: governance.LibraryVerificationFailed, + SnapshotID: "snap-2", JobID: "job2", Failure: "恢复后内容仍与预备份不一致", + StartedAt: time.Now().UTC().Add(-time.Minute), FinishedAt: time.Now().UTC(), + }}} + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil) + libraryVerifications(recorder, request, Config{Store: store}) + body := decodeBody(t, recorder) + if body["verified"] != false { + t.Fatalf("failed evidence must not verify: %v", body) + } + if reason, _ := body["reason"].(string); !strings.Contains(reason, "恢复后内容仍与预备份不一致") { + t.Fatalf("reason=%q", reason) + } +} + +// 同步闸口必须消费同一 target 的证据：有 passed 证据即放行， +// 只有失败证据时仍然拒绝并给出失败原因（不是笼统"没有证据"）。 +func TestLibrarySyncGateConsumesRestoreEvidence(t *testing.T) { + base := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false, + LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40} + base.Store = &verificationStore{verifications: []governance.LibraryRestoreVerification{{ + Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1", + JobID: "job1", FinishedAt: time.Now().UTC(), + }}} + allowed := librarySyncGateWithGrant(base.withRestoreEvidence(context.Background(), "internal"), + libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true) + if !allowed.SyncAllowed { + t.Fatalf("gate should allow with passed evidence: %v", allowed.Reasons) + } + noEvidence := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false, + LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40, + Store: &verificationStore{}}.withRestoreEvidence(context.Background(), "internal") + blocked := librarySyncGateWithGrant(noEvidence, + libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true) + if blocked.SyncAllowed { + t.Fatalf("gate without env and without evidence must block: %v", blocked.Reasons) + } + if len(blocked.Reasons) == 0 || !strings.Contains(strings.Join(blocked.Reasons, "；"), "恢复验证") { + t.Fatalf("blocked reasons=%v", blocked.Reasons) + } + failedConfig := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false, + LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40, + Store: &verificationStore{verifications: []governance.LibraryRestoreVerification{{ + Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-2", + JobID: "job2", Failure: "canary 后校验仍通过", FinishedAt: time.Now().UTC(), + }}}} + reason := strings.Join(restoreGateReasons(failedConfig.withRestoreEvidence(context.Background(), "internal")), "；") + if !strings.Contains(reason, "canary 后校验仍通过") { + t.Fatalf("失败证据的失败原因必须出现在闸口：%q", reason) + } +} diff --git a/internal/httpapi/production_bindings.go b/internal/httpapi/production_bindings.go index 7e72939..43756c7 100644 --- a/internal/httpapi/production_bindings.go +++ b/internal/httpapi/production_bindings.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "knowledge-governance/internal/pgdsn" "knowledge-governance/internal/platform/weknora" ) @@ -33,6 +34,55 @@ type productionBindingView struct { ProductionBaseURL string `json:"production_base_url"` ProductionConnected bool `json:"production_connected"` ProductionMessage string `json:"production_message"` + // LegacyScope 说明这条记录属于哪条发布路径。整库发布（封存/同步/恢复）不读它， + // 只读下面的 WholeTarget；页面必须把两者分开写，否则会把旧文档级目标当成整库目标。 + LegacyScope string `json:"legacy_scope"` + // WholeTarget 是整库发布真正写入的目标（来自部署环境 LIBRARY_PRODUCTION_*）， + // 与上面可编辑的文档级绑定是两条独立路径。 + WholeTarget libraryPublishTargetView `json:"whole_target"` +} + +// libraryPublishTargetView 是"整库发布到底写哪儿"的只读投影。 +// 为什么由后端给出：页面上的 kb 名称/文档数来自生产端列表接口，而整库写入用的是 +// 另一组环境配置；让页面自己拼会把两条路径混成一个。 +type libraryPublishTargetView struct { + Configured bool `json:"configured"` + Source string `json:"source"` + TenantID uint64 `json:"tenant_id"` + InternalKBID string `json:"internal_kb_id"` + ExternalKBID string `json:"external_kb_id"` + Database string `json:"database"` + ElasticIndex string `json:"elastic_index"` + ObjectBucket string `json:"object_bucket"` + MissingReason string `json:"missing_reason"` +} + +// libraryPublishTarget 从 config 派生整库目标；缺项照实说缺哪一项，不猜默认值。 +func (config Config) libraryPublishTarget() libraryPublishTargetView { + view := libraryPublishTargetView{ + Source: "部署环境 LIBRARY_PRODUCTION_*", + TenantID: config.LibraryProductionTenantID, + InternalKBID: config.LibraryProductionInternalKB, + ExternalKBID: config.LibraryProductionExternalKB, + Database: pgdsn.Endpoint(config.LibraryProductionDatabaseURL), + ElasticIndex: config.LibraryProductionIndex, + ObjectBucket: config.LibraryProductionS3Bucket, + } + missing := make([]string, 0, 4) + if config.LibraryProductionDatabaseURL == "" { + missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL") + } + if view.InternalKBID == "" { + missing = append(missing, "LIBRARY_PRODUCTION_KB_INTERNAL") + } + if config.LibraryProductionIndex == "" { + missing = append(missing, "LIBRARY_PRODUCTION_INDEX") + } + view.Configured = len(missing) == 0 + if len(missing) > 0 { + view.MissingReason = "未配置：" + strings.Join(missing, "、") + } + return view } // productionTargetBaseURL is the production endpoint the picker reads from: the @@ -79,6 +129,10 @@ func (config Config) productionBindingPayload(ctx context.Context) (productionBi InternalKBName: names[storedInternal], ExternalKBName: names[storedExternal], UpdatedBy: updatedBy, ProductionBaseURL: config.productionTargetBaseURL(ctx), ProductionConnected: message == "", ProductionMessage: message, + // 这张表服务的是"文档级发布"（单篇文档写生产）。整库发布读的是部署环境 + // LIBRARY_PRODUCTION_*，两者不能互相顶替，所以页面必须分开显示。 + LegacyScope: "文档级发布（单篇文档写生产）· 旧路径", + WholeTarget: config.libraryPublishTarget(), }, nil } diff --git a/internal/httpapi/production_bindings_test.go b/internal/httpapi/production_bindings_test.go index db5a77f..229a597 100644 --- a/internal/httpapi/production_bindings_test.go +++ b/internal/httpapi/production_bindings_test.go @@ -178,3 +178,50 @@ func TestProductionBindingOptionsReportUnavailableReason(t *testing.T) { t.Fatalf("status = %d body %s", recorder.Code, recorder.Body.String()) } } + +// 页面必须说清"整库发布真正写哪个 KB/端点"。这条投影只能来自部署配置， +// 不能从可编辑的文档级绑定推出来，也不能把口令带进响应。 +func TestLibraryPublishTargetIsDerivedFromDeploymentConfig(t *testing.T) { + config := Config{ + // 只放形状，不放任何可用凭据：口令段是占位符，主机是 .invalid。 + LibraryProductionDatabaseURL: "postgres://governance@db.invalid:5432/weknora_test?sslmode=disable", + LibraryProductionInternalKB: "a8e9c8ad-411c-4c53-8421-599d29e9b179", + LibraryProductionExternalKB: "2e980686-2233-4c4e-b5a6-8b0069477b8e", + LibraryProductionTenantID: 10000, + LibraryProductionIndex: "weknora_test_chunks", + LibraryProductionS3Bucket: "dobot-weknora-test", + } + target := config.libraryPublishTarget() + if !target.Configured || target.MissingReason != "" { + t.Fatalf("完整配置应判为已配置: %+v", target) + } + if target.InternalKBID != "a8e9c8ad-411c-4c53-8421-599d29e9b179" || target.TenantID != 10000 { + t.Fatalf("必须回显实际整库目标而不是占位: %+v", target) + } + if target.Database != "db.invalid:5432/weknora_test" { + t.Fatalf("端点应只含 host:port/db，得到 %q", target.Database) + } + // 回显只能是端点：连接串里的用户信息（口令、账号）一个字符都不能进响应。 + if strings.Contains(target.Database, "@") || strings.Contains(target.Source, "@") || + strings.Contains(target.Source, "postgres://") { + t.Fatalf("响应里不能出现连接串用户信息: %+v", target) + } + if target.ElasticIndex != "weknora_test_chunks" || target.ObjectBucket != "dobot-weknora-test" { + t.Fatalf("索引与桶也要显示: %+v", target) + } +} + +func TestLibraryPublishTargetNamesMissingConfigInsteadOfGuessing(t *testing.T) { + target := Config{}.libraryPublishTarget() + if target.Configured { + t.Fatal("缺配置时不能报告已配置") + } + for _, key := range []string{"LIBRARY_PRODUCTION_DATABASE_URL", "LIBRARY_PRODUCTION_KB_INTERNAL"} { + if !strings.Contains(target.MissingReason, key) { + t.Fatalf("缺失说明应点名 %s，得到 %q", key, target.MissingReason) + } + } + if target.InternalKBID != "" || target.Database != "（未配置）" { + t.Fatalf("缺配置就不要编造目标: %+v", target) + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index f6f6d1c..87ac1ca 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -181,6 +181,9 @@ type Config struct { LibraryProductionInternalKB string LibraryProductionExternalKB string LibraryProductionTenantID uint64 + // 整库发布还落在这些后端上；页面要显示"到底写哪儿"，只给 KB ID 不够。 + LibraryProductionIndex string + LibraryProductionS3Bucket string // LibraryScopeProbe 让测试注入"目标整库是否为空"的探测；留空即用真实连接。 // 它不是运行时开关：生产部署不设置它，那次探测仍然直连目标库逐表计数。 LibraryScopeProbe func(ctx context.Context, scope string) (bool, map[string]int64, error) @@ -198,8 +201,13 @@ type Config struct { RequireWeKnoraReady bool ProductionReleaseRequestsEnabled bool // TestAutoReviewPolicy 只用于展示：让运维在界面上看到"现在自动批准到什么程度"。 - TestAutoReviewPolicy string - ProductionBackupRestoreVerified bool + TestAutoReviewPolicy string + ProductionBackupRestoreVerified bool + // restoreVerified / restoreEvidenceReason 由请求期的证据读取填入（withRestoreEvidence）： + // 闸口按"当前目标是否有真实恢复验证证据"判断，而不是按可手填的 env。 + // 它们不可由配置注入，避免把证据变成另一个开关。 + restoreVerified bool + restoreEvidenceReason string ProductionReleaseAuthorizerRef string ProductionReleaseAuthorizationToken string // DingtalkSources 枚举「我账号下能看到」的钉钉知识库与钉盘空间（只读，dws）。 @@ -556,6 +564,14 @@ func NewHandler(config Config) (http.Handler, error) { mux.HandleFunc("POST /api/v1/library/baselines", func(w http.ResponseWriter, r *http.Request) { libraryBaselineEnqueue(w, r, config) }) + // 恢复验证：真实备份 → canary → 恢复 → 复核，通过后证据按当前目标登记， + // 同步闸口据此放行（不再需要页面/环境变量手填 verified）。 + mux.HandleFunc("POST /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) { + libraryVerifyEnqueue(w, r, config) + }) + mux.HandleFunc("GET /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) { + libraryVerifications(w, r, config) + }) // 全部 main 提交时间线（读本库缓存，不直接打 Gitea） mux.HandleFunc("GET /api/v1/version-history", func(w http.ResponseWriter, r *http.Request) { versionHistoryList(w, r, config) diff --git a/cmd/librarydrill/canary_test.go b/internal/libraryverify/canary_test.go similarity index 99% rename from cmd/librarydrill/canary_test.go rename to internal/libraryverify/canary_test.go index 1b9c156..6ad69f1 100644 --- a/cmd/librarydrill/canary_test.go +++ b/internal/libraryverify/canary_test.go @@ -1,4 +1,4 @@ -package main +package libraryverify import ( "context" diff --git a/internal/libraryverify/verify.go b/internal/libraryverify/verify.go new file mode 100644 index 0000000..8c88ea2 --- /dev/null +++ b/internal/libraryverify/verify.go @@ -0,0 +1,266 @@ +// Package libraryverify 在目标实例上做一次**真实**的恢复验证（原 cmd/librarydrill 的实现）： +// +// 维护隔离（关入口/停写入者/证明排空） +// → 捕获目标整库（真预备份，落归档） +// → 校验目标 == 预备份（基线一致） +// → 写入一个 canary（证明"校验确实能发现内容变化"） +// → 按预备份恢复 +// → 逐表/索引/资源/Wiki 再校验一次，必须完全一致 +// → 关窗口（按原运行状态恢复服务、开入口、验证健康） +// +// 为什么搬进 internal：页面要能点一次"恢复验证"，worker 必须在进程内执行同一份实现， +// 不能再要求运维 SSH 上去跑 CLI。命令行使同一个 Run，不允许两份实现。 +// 它只读/写**被测目标自己**，绝不触碰封存源；所有凭据来自进程环境，不进命令行、不打印。 +// 任一步失败都不伪装成功：窗口保持维护、报告写明失败阶段。 +package libraryverify + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "log" + "time" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/libraryfreeze" + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarystack" + v071 "knowledge-governance/internal/releaseadapter/v071" +) + +type Verification struct { + SnapshotID string `json:"snapshot_id"` + Verified bool `json:"verified"` + Error string `json:"error,omitempty"` + At time.Time `json:"at"` +} + +type Report struct { + Status string `json:"status"` // passed | failed | maintenance_held + Scope string `json:"scope"` + Target string `json:"target"` + SnapshotID string `json:"snapshot_id"` + Project string `json:"project"` + Counts map[string]int64 `json:"counts"` + BaselineCheck *Verification `json:"baseline_check"` + DriftCheck *Verification `json:"drift_check"` // 必须失败：证明确实能发现差异 + RestoreCheck *Verification `json:"restore_check"` + Canary string `json:"canary"` + Failure string `json:"failure,omitempty"` + Stage string `json:"stage,omitempty"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + Until map[string]string `json:"until,omitempty"` +} + +// Progress 把演练的阶段推进交给调用方（worker 写 library_job_stages，CLI 打日志）。 +type Progress func(stage, state, detail string) + +func (p Progress) report(stage, state, detail string) { + if p != nil { + p(stage, state, detail) + } +} + +// Run 执行一次演练。jobID 同时是维护窗口的持有者标识，必须与业务作业 id 一致， +// 否则崩溃恢复时窗口与作业对不上。 +func Run(ctx context.Context, config appconfig.Config, scope, jobID string, progress Progress) (*Report, error) { + if scope != "internal" && scope != "external" { + return nil, errors.New("scope must be internal or external") + } + report := &Report{Status: "failed", Scope: scope, Project: config.LibraryTargetProject, + Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()} + fail := func(stage string, cause error, held bool) (*Report, error) { + report.Stage = stage + report.Failure = cause.Error() + report.FinishedAt = time.Now().UTC() + if held { + report.Status = "maintenance_held" + } + progress.report(stage, "failed", cause.Error()) + return report, cause + } + + progress.report("capture", "running", "打开目标栈与维护管理器") + target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope) + if err != nil { + return fail("capture", err, false) + } + defer target.Close() + manager, err := librarystack.OpenMaintenance(config) + if err != nil { + return fail("capture", err, false) + } + defer manager.Close() + if err := manager.VerifyTopology(ctx); err != nil { + return fail("capture", err, false) + } + coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot) + if err != nil { + return fail("capture", err, false) + } + defer control.Close() + defer coordinator.Close() + + var ( + snapshotID string + counts map[string]int64 + canary string + contentTouched bool + baseline Verification + drift Verification + restored Verification + ) + work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) { + start := time.Now() + current, err := target.Current(ctx) + if err != nil { + return "", errors.New("读取目标当前整库状态失败：" + err.Error()) + } + counts = current.Counts + report.Counts = counts + // 1) 真预备份（落归档、内容寻址、导出后复核内容未变）。 + backup, err := backups.Prepare(ctx, "", current) + if err != nil { + return "", errors.New("捕获目标整库失败：" + err.Error()) + } + snapshotID = backup.SnapshotID + report.SnapshotID = snapshotID + if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil { + return "", errors.New("记录预备份身份失败：" + err.Error()) + } + progress.report("capture", "succeeded", + fmt.Sprintf("已捕获目标整库快照 %s（%d 字节，用时 %s）", snapshotID, backup.Bytes, time.Since(start).Round(time.Second))) + log.Printf("verify: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start)) + + // 2) 基线校验：预备份必须与目标完全一致。 + progress.report("verify", "running", "逐表/索引/资源比对目标与预备份") + baseline = Verification{SnapshotID: snapshotID, At: time.Now().UTC()} + if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil { + baseline.Error = err.Error() + report.BaselineCheck = &baseline + return "", errors.New("预备份与目标不一致（预备份本身不可信）：" + err.Error()) + } + baseline.Verified = true + report.BaselineCheck = &baseline + progress.report("verify", "succeeded", "预备份与目标逐类一致") + + // Canary 是隔离的标签行，空目标上也一样：绝不为了"让演练能跑"去种文档或改既有行。 + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return "", err + } + canary = fmt.Sprintf("%x", random[:]) + identityValue := target.Identity() + var exists bool + if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil { + return "", err + } + if exists { + return "", errors.New("canary identity collision; no write attempted") + } + ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope, + TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID, + BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID, + CreatedAt: time.Now().UTC(), Universe: backup.Universe} + report.Canary = canary + progress.report("canary", "running", "写入一个隔离 canary，验证校验能发现差异") + if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary, + func(ctx context.Context) error { + if err := window.MarkContentRisk(ctx); err != nil { + return err + } + contentTouched = true + return nil + }, func(ctx context.Context) error { + _, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary) + return err + }); err != nil { + return "", fmt.Errorf("canary preparation/write failed: %w", err) + } + drift = Verification{SnapshotID: snapshotID, At: time.Now().UTC()} + if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil { + drift.Error = "写入 canary 后校验仍然通过：校验没有真正比对内容" + report.DriftCheck = &drift + return "", errors.New("写入 canary 后校验仍然通过：校验没有真正比对内容，拒绝把这次演练当成通过") + } else { + drift.Error = verifyErr.Error() + } + report.DriftCheck = &drift + progress.report("canary", "succeeded", "校验按预期发现了 canary 差异") + log.Printf("verify: drift detected as expected (%s)", drift.Error) + + // 4) 按预备份恢复。恢复只删掉持久写集合里那个新 canary 身份。 + progress.report("restore", "running", "按预备份恢复目标整库") + if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID, + WriteSet: &ledger}); err != nil { + return "", errors.New("按预备份恢复失败：" + err.Error()) + } + // 5) 恢复后逐类复核：必须与预备份完全一致。 + restored = Verification{SnapshotID: snapshotID, At: time.Now().UTC()} + if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil { + restored.Error = err.Error() + report.RestoreCheck = &restored + return "", errors.New("恢复后内容仍与预备份不一致：" + err.Error()) + } + restored.Verified = true + report.RestoreCheck = &restored + progress.report("restore", "succeeded", "恢复后与预备份逐类一致") + log.Printf("verify: restored and verified snapshot=%s", snapshotID) + return snapshotID, nil + } + + progress.report("finalize", "running", "关闭维护窗口并按原状态恢复目标") + window, windowErr := coordinator.RunWindow(ctx, jobID, + func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work, + // 没写过目标内容 ⇒ 按原状态恢复永远安全；写过 canary ⇒ 只有恢复并校验通过才开放。 + func(error) bool { return !contentTouched || restored.Verified }) + + report.Until = map[string]string{"job": jobID, "window_state": window.State} + if windowErr != nil || window.State != "succeeded" { + cause := windowErr + if cause == nil { + cause = fmt.Errorf("维护窗口状态 %s（未成功）", window.State) + } + if window.State == "held" { + return fail("finalize", cause, true) + } + return fail("finalize", cause, false) + } + report.Status = "passed" + report.FinishedAt = time.Now().UTC() + progress.report("finalize", "succeeded", "维护窗口已关闭，目标恢复原运行状态") + return report, nil +} + +func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error { + ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false}) + if err := ledger.Seal(); err != nil { + return err + } + if _, err := v071.PersistWriteSet(root, ledger); err != nil { + return err + } + if err := mark(ctx); err != nil { + return err + } + return write(ctx) +} + +// RecoverWindows 把非终态的维护窗口按原运行状态恢复（服务起来、入口验证）。 +// 演练失败但还没写目标内容时，用它把目标从"停住"状态救回来；治理记录由作业/接口那侧负责。 +func RecoverWindows(config appconfig.Config) error { + manager, err := librarystack.OpenMaintenance(config) + if err != nil { + return err + } + defer manager.Close() + coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot) + if err != nil { + return err + } + defer control.Close() + defer coordinator.Close() + return coordinator.Recover(context.Background()) +} diff --git a/internal/platform/postgres/library_dispatch.go b/internal/platform/postgres/library_dispatch.go index 3cc3377..f3c0330 100644 --- a/internal/platform/postgres/library_dispatch.go +++ b/internal/platform/postgres/library_dispatch.go @@ -33,7 +33,8 @@ func (s *Store) createLibraryDispatch(ctx context.Context, job governance.Librar return created, true, nil } kind := "library." + string(created.Kind) - if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore && created.Kind != governance.LibraryJobBaseline { + if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore && + created.Kind != governance.LibraryJobBaseline && created.Kind != governance.LibraryJobVerify { return created, reused, errors.New("unsupported library dispatch kind") } queueID, kind, raw, err := governance.LibraryQueueRequest(handoff, created.ID, created.Scope, created.TargetSnapshotID, kind) diff --git a/internal/platform/postgres/library_verification.go b/internal/platform/postgres/library_verification.go new file mode 100644 index 0000000..e42ab37 --- /dev/null +++ b/internal/platform/postgres/library_verification.go @@ -0,0 +1,98 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +// 恢复验证证据：只由 worker 在真实演练结束后写入；这里只做落库与按当前目标读取。 +// +// 读取永远带 target_id = 当前部署目标：换库/换索引/换桶之后旧证据自动失效， +// 不需要任何人去"记得把开关关掉"。 + +// SaveLibraryRestoreVerification 幂等登记一次演练结果（同一 target+scope+job 只留一条）。 +func (s *Store) SaveLibraryRestoreVerification(ctx context.Context, verification governance.LibraryRestoreVerification) error { + if verification.ID == "" || verification.JobID == "" { + return fmt.Errorf("restore verification requires id and job id") + } + if verification.Scope != "internal" && verification.Scope != "external" { + return fmt.Errorf("restore verification scope %q is invalid", verification.Scope) + } + if verification.Status != governance.LibraryVerificationPassed && verification.Status != governance.LibraryVerificationFailed { + return fmt.Errorf("restore verification status %q is invalid", verification.Status) + } + report, err := json.Marshal(verification.Report) + if err != nil { + return fmt.Errorf("encode restore verification report: %w", err) + } + _, err = s.db.Exec(ctx, ` +INSERT INTO library_restore_verifications (id, target_id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) +ON CONFLICT (target_id, scope, job_id) DO UPDATE +SET status = EXCLUDED.status, snapshot_id = EXCLUDED.snapshot_id, failure = EXCLUDED.failure, + report = EXCLUDED.report, started_at = EXCLUDED.started_at, finished_at = EXCLUDED.finished_at`, + verification.ID, s.libraryTargetID, verification.Scope, verification.Status, + verification.SnapshotID, verification.JobID, verification.Failure, report, + verification.StartedAt.UTC(), verification.FinishedAt.UTC()) + if err != nil { + return fmt.Errorf("save restore verification: %w", err) + } + return nil +} + +// LatestLibraryRestoreVerification 按完成时间取当前目标该 scope 的最近一条证据。 +// 没有记录返回 (zero,false,nil)：那不是错误，页面据此说"还没有验证过"。 +func (s *Store) LatestLibraryRestoreVerification(ctx context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) { + var ( + item governance.LibraryRestoreVerification + report []byte + ) + err := s.db.QueryRow(ctx, ` +SELECT id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at +FROM library_restore_verifications +WHERE target_id=$1 AND scope=$2 +ORDER BY finished_at DESC, created_at DESC +LIMIT 1`, s.libraryTargetID, scope).Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID, + &item.JobID, &item.Failure, &report, &item.StartedAt, &item.FinishedAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryRestoreVerification{}, false, nil + } + return governance.LibraryRestoreVerification{}, false, fmt.Errorf("load restore verification: %w", err) + } + item.Report = decodeEvidenceObject(report) + return item, true, nil +} + +// LibraryRestoreVerifications 是页面用的轻量清单（最近 N 条），按目标过滤。 +func (s *Store) LibraryRestoreVerifications(ctx context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) { + if limit <= 0 { + limit = 10 + } + rows, err := s.db.Query(ctx, ` +SELECT id, scope, status, snapshot_id, job_id, failure, started_at, finished_at +FROM library_restore_verifications +WHERE target_id=$1 AND scope=$2 +ORDER BY finished_at DESC, created_at DESC +LIMIT $3`, s.libraryTargetID, scope, limit) + if err != nil { + return nil, fmt.Errorf("list restore verifications: %w", err) + } + defer rows.Close() + items := make([]governance.LibraryRestoreVerification, 0, limit) + for rows.Next() { + var item governance.LibraryRestoreVerification + if err := rows.Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID, &item.JobID, + &item.Failure, &item.StartedAt, &item.FinishedAt); err != nil { + return nil, err + } + items = append(items, item) + } + return items, rows.Err() +} diff --git a/internal/platform/postgres/migrations/000079_library_restore_verification.down.sql b/internal/platform/postgres/migrations/000079_library_restore_verification.down.sql new file mode 100644 index 0000000..ea41c58 --- /dev/null +++ b/internal/platform/postgres/migrations/000079_library_restore_verification.down.sql @@ -0,0 +1,13 @@ +DROP INDEX IF EXISTS library_restore_verifications_latest; +DROP TABLE IF EXISTS library_restore_verifications; + +-- 先把 verify 作业清掉（它没有历史数据，只可能来自本版本），再恢复旧的 kind 约束。 +DELETE FROM library_jobs WHERE kind = 'verify'; +ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check; +ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check + CHECK (kind IN ('seal','sync','restore','baseline')); + +DROP INDEX IF EXISTS library_jobs_write_target; +CREATE UNIQUE INDEX library_jobs_write_target + ON library_jobs(scope) + WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline'); \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000079_library_restore_verification.up.sql b/internal/platform/postgres/migrations/000079_library_restore_verification.up.sql new file mode 100644 index 0000000..72a7398 --- /dev/null +++ b/internal/platform/postgres/migrations/000079_library_restore_verification.up.sql @@ -0,0 +1,37 @@ +-- 恢复验证成为一种独立业务作业，并且它的证据只能由真实演练写入。 +-- +-- 为什么需要：闸门原来读 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个纯 env 开关， +-- 谁把它写成 true 都能"证明"生产可恢复；页面也没有入口去真跑一次恢复。 +-- 现在 verified 只能来自 library_restore_verifications 里 passed 的真实记录。 +-- +-- verify 与 sync/restore/baseline 共用同一把写互斥：它同样要停写入者、开维护窗口、 +-- 在目标上写 canary 再恢复，不能和别的写作业并发。 +ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check; +ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check + CHECK (kind IN ('seal','sync','restore','baseline','verify')); + +DROP INDEX IF EXISTS library_jobs_write_target; +CREATE UNIQUE INDEX library_jobs_write_target + ON library_jobs(scope) + WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline','verify'); + +-- target_id 是 appconfig.LibraryTargetIdentity()：换目标（库/索引/桶/实例）后旧证据不再适用， +-- 这是"证据必须对得上当前目标"的落库形态。 +CREATE TABLE IF NOT EXISTS library_restore_verifications ( + id text PRIMARY KEY, + target_id text NOT NULL CHECK (target_id <> ''), + scope text NOT NULL CHECK (scope IN ('internal','external')), + status text NOT NULL CHECK (status IN ('passed','failed')), + snapshot_id text NOT NULL DEFAULT '', + job_id text NOT NULL CHECK (job_id <> ''), + failure text NOT NULL DEFAULT '', + report jsonb, + started_at timestamptz NOT NULL, + finished_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + -- 同一作业只留一条；重复上报是幂等重放，不是新证据。 + CONSTRAINT library_restore_verifications_job UNIQUE (target_id, scope, job_id) +); + +CREATE INDEX IF NOT EXISTS library_restore_verifications_latest + ON library_restore_verifications (target_id, scope, status, finished_at DESC); \ No newline at end of file diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index d9d36c1..402bde7 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,8 +11,15 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 78 { - t.Fatalf("migration count = %d, want 78", len(migrations)) + if len(migrations) != 79 { + t.Fatalf("migration count = %d, want 79", len(migrations)) + } + // 79 是恢复验证：verify 成为独立业务作业并与写作业共用互斥， + // 证据按 target_id 落库（换目标旧证据自动失效）。 + if verification := migrations[78]; verification.Version != 79 || + !strings.Contains(verification.UpSQL, "library_restore_verifications") || + !strings.Contains(verification.UpSQL, "'verify'") { + t.Fatalf("migration 79 = %#v", verification) } // 78 是 DS01 恢复标记派发台账：唯一 key、单调状态与激活登记；只动治理库。 if dispatch := migrations[77]; dispatch.Version != 78 || diff --git a/web/src/BackupScheduleSection.test.tsx b/web/src/BackupScheduleSection.test.tsx index a694399..dd9fa32 100644 --- a/web/src/BackupScheduleSection.test.tsx +++ b/web/src/BackupScheduleSection.test.tsx @@ -22,6 +22,7 @@ const base: BackupSchedule = { last_applied_summary: '每天 03:30 · 保留 7 天 · 每月归档 3 份', host_configured: true, host_target: 'weknora-test', + host_endpoint: '100.85.111.121:22', host_reachable: true, host_error: '', host_schedule_enabled: true, @@ -88,6 +89,9 @@ describe('BackupScheduleView', () => { '重新下发当前设置', '立即备份一次', 'weknora-test · 已连接', + '拨号地址', + '100.85.111.121:22', + '全库，不是 scope 快照', '每天 03:30 · 保留 7 天 · 每月 3 份', '已一致', '20261009T052909Z', @@ -120,6 +124,28 @@ describe('BackupScheduleView', () => { expect(html).toContain('disabled') }) + it('连不上主机：报连接失败与拨号地址，脚本状态写未知而不是"缺失"', () => { + const html = view({ + host_reachable: false, + host_endpoint: '120.76.217.51:22', + host_error: + '目标主机 weknora-test 不可达（ssh: connect to host 120.76.217.51 port 22: Operation timed out）；未执行任何主机侧脚本，脚本与计划状态未知（拨号地址 120.76.217.51:22）', + in_sync: false, + host_script_present: false, + }) + expect(html).toContain('weknora-test · 连接失败') + expect(html).toContain('120.76.217.51:22') + expect(html).toContain('未知（未连上主机，未执行主机侧脚本）') + expect(html).not.toContain('缺失') + }) + + it('已连上且脚本真的不在：这时才显示"缺失"', () => { + const html = view({ host_reachable: true, host_script_present: false, in_sync: false }) + expect(html).toContain('weknora-test · 已连接') + expect(html).toContain('缺失') + expect(html).not.toContain('未知（未连上主机') + }) + it('待下发（主机上的计划与设置不一致）时提示', () => { const html = view({ host_schedule_time: '05:00', in_sync: false }) expect(html).toContain('待下发') diff --git a/web/src/BackupScheduleSection.tsx b/web/src/BackupScheduleSection.tsx index 029efef..3ba67a8 100644 --- a/web/src/BackupScheduleSection.tsx +++ b/web/src/BackupScheduleSection.tsx @@ -91,6 +91,12 @@ export function BackupScheduleView(props: BackupScheduleViewProps) { : schedule.host_reachable ? `${schedule.host_target} · 已连接` : `${schedule.host_target} · 连接失败` + // 只有真的连上并读回主机状态时，才能说脚本在不在；连不上是"未知"。 + const scriptLine = !schedule.host_reachable + ? '未知（未连上主机，未执行主机侧脚本）' + : schedule.host_script_present + ? '存在' + : '缺失' return (
@@ -115,11 +121,12 @@ export function BackupScheduleView(props: BackupScheduleViewProps) {
目标主机
{hostLine}
+
拨号地址
+
{schedule.host_endpoint || '—'}
+
备份脚本
+
{scriptLine}
主机 cron
-
- {schedule.host_cron_service ? `服务 ${schedule.host_cron_service}` : '—'} - {schedule.host_script_present ? '' : ' · 备份脚本缺失'} -
+
{schedule.host_cron_service ? `服务 ${schedule.host_cron_service}` : '—'}
最近一次备份
{schedule.last_run_stamp || '—'} @@ -213,8 +220,12 @@ export function BackupScheduleView(props: BackupScheduleViewProps) { {busy === 'apply' ? '正在下发…' : '重新下发当前设置'} +

+ 「立即备份一次」执行的是主机上的整库逻辑备份（该主机上全部业务库的 pg_dump 目录）， + 不是本次对内/对外 scope 的整库快照；scope 快照在「发布管理 → 整库版本」里封存。 +

{message ? (

{message} diff --git a/web/src/ProductionBindingsSection.test.tsx b/web/src/ProductionBindingsSection.test.tsx new file mode 100644 index 0000000..7587f24 --- /dev/null +++ b/web/src/ProductionBindingsSection.test.tsx @@ -0,0 +1,101 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe, expect, it, vi } from 'vitest' + +import { ProductionBindingsView } from './ProductionBindingsSection' +import type { ProductionBindings } from './productionBindings' + +const bindings = (overrides: Partial = {}): ProductionBindings => ({ + internal_kb_id: '0b217e8a-2a65-40a3-ab8f-34ac5c7a6f05', + external_kb_id: '9a1e5c11-0000-4000-8000-000000000002', + internal_kb_name: '对内知识库-wiki', + external_kb_name: '对外知识库-wiki', + updated_by: 'yaozhanghao', + production_base_url: 'https://weknora-test.dobot.cn', + production_connected: true, + production_message: '', + legacy_scope: '文档级发布（单篇文档写生产）· 旧路径', + whole_target: { + configured: true, + source: '部署环境 LIBRARY_PRODUCTION_*', + tenant_id: 10000, + internal_kb_id: 'a8e9c8ad-411c-4c53-8421-599d29e9b179', + external_kb_id: '2e980686-2233-4c4e-b5a6-8b0069477b8e', + database: 'pgm-wz9ohr9gk919pr1q.pg.rds.aliyuncs.com:5432/weknora_test', + elastic_index: 'weknora_test_chunks', + object_bucket: 'dobot-weknora-test', + missing_reason: '', + }, + ...overrides, +}) + +const view = (value: ProductionBindings | null, admin = true) => + renderToStaticMarkup( + + ) + +describe('ProductionBindingsView', () => { + it('把实际整库目标（完整 ID + 端点）显示出来，而不是只显示可编辑的文档级绑定', () => { + const html = view(bindings()) + expect(html).toContain('整库发布目标（实际写入，只读）') + // 完整 ID 必须可读：截断会让操作者核对不了 a8e9c8ad 与 0b217e8a 的区别。 + expect(html).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179') + expect(html).toContain('2e980686-2233-4c4e-b5a6-8b0069477b8e') + expect(html).toContain('pgm-wz9ohr9gk919pr1q.pg.rds.aliyuncs.com:5432/weknora_test') + expect(html).toContain('weknora_test_chunks') + expect(html).toContain('dobot-weknora-test') + expect(html).toContain('部署环境 LIBRARY_PRODUCTION_*') + }) + + it('旧文档级绑定标成旧路径，不再声称"发布写入此库"', () => { + const html = view(bindings()) + expect(html).toContain('文档级发布（单篇文档写生产）· 旧路径') + expect(html).toContain('仅文档级发布读取这组绑定；整库发布（封存/同步/恢复）不读它。') + expect(html).toContain('对内文档级目标') + expect(html).not.toContain('发布写入此库') + }) + + it('整库目标缺配置：点名缺哪个环境变量，且不拿文档级绑定顶上去', () => { + const html = view( + bindings({ + whole_target: { + configured: false, + source: '部署环境 LIBRARY_PRODUCTION_*', + tenant_id: 0, + internal_kb_id: '', + external_kb_id: '', + database: '（未配置）', + elastic_index: '', + object_bucket: '', + missing_reason: '未配置：LIBRARY_PRODUCTION_DATABASE_URL、LIBRARY_PRODUCTION_KB_INTERNAL', + }, + }) + ) + expect(html).toContain('LIBRARY_PRODUCTION_DATABASE_URL') + expect(html).toContain('整库目标配置不全') + // 文档级绑定仍然显示，但必须仍是"文档级"身份，不能冒充整库目标。 + expect(html).toContain('对内文档级目标') + }) + + it('非管理员看得到目标，但没有保存入口', () => { + const html = view(bindings(), false) + expect(html).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179') + expect(html).not.toContain('保存文档级目标') + expect(html).toContain('仅管理员可修改文档级发布目标') + }) +}) diff --git a/web/src/ProductionBindingsSection.tsx b/web/src/ProductionBindingsSection.tsx index abcfef6..17c62a3 100644 --- a/web/src/ProductionBindingsSection.tsx +++ b/web/src/ProductionBindingsSection.tsx @@ -5,8 +5,182 @@ import { fetchProductionBindingOptions, fetchProductionBindings, saveProductionB import type { ProductionBindings } from './productionBindings' import type { KnowledgeBaseOption } from './routingBindings' -// 「生产知识库绑定」：发布链路的 源库 → 目标库 映射。这里选的是发布要写入的 -// 生产（目标）知识库，与上面的「测试知识库绑定」（测试端收件库）是两件事。 +// 「生产知识库绑定」：这里同时涉及两条独立路径，页面必须分开写清楚： +// 1) 整库发布目标（封存/同步/恢复真正写入的库与端点）——来自部署配置，只读； +// 2) 文档级绑定（旧流程：单篇文档发布写哪个库）——可在这里编辑。 +// 把 2 写成"发布写入此库"会让操作者以为整库发布会写进 0b217e8a 这类 2000+ 文档库。 + +export interface ProductionBindingsViewProps { + bindings: ProductionBindings | null + options: KnowledgeBaseOption[] + internalDraft: string + externalDraft: string + isAdmin: boolean + loading: boolean + saving: boolean + error: string | null + message: string + dirty: boolean + sameSelection: boolean + onInternalChange: (next: string) => void + onExternalChange: (next: string) => void + onSave: () => void + onReload: () => void +} + +export function ProductionBindingsView(props: ProductionBindingsViewProps) { + const { + bindings, + options, + internalDraft, + externalDraft, + isAdmin, + loading, + saving, + error, + message, + dirty, + sameSelection, + onInternalChange, + onExternalChange, + onSave, + onReload, + } = props + const optionLabel = (base: KnowledgeBaseOption) => `${base.name}（${base.knowledge_count} 文档）` + const renderSide = (label: string, boundID: string, boundName: string) => ( +

+ {label} + + {boundID ? boundName || `${boundID.slice(0, 8)}…` : '未绑定 · 按同 ID 语义发布（目标端须存在同 ID 库）'} + + {boundID ? '文档级绑定 · 只影响单篇文档发布，不改整库目标' : '未绑定：沿用测试库 ID'} +
+ ) + const whole = bindings?.whole_target + const wholeRows: [string, string][] = whole + ? [ + ['对内目标知识库', whole.internal_kb_id || '未配置'], + ['对外目标知识库', whole.external_kb_id || '未配置'], + ['目标数据库', whole.database || '—'], + ['检索索引', whole.elastic_index || '—'], + ['对象存储桶', whole.object_bucket || '—'], + ['租户', whole.tenant_id ? String(whole.tenant_id) : '—'], + ['配置来源', whole.source || '—'], + ] + : [] + const editorFor = (value: string, onChange: (next: string) => void, placeholder: string) => + options.length > 0 ? ( + + ) : ( + onChange(event.target.value.trim())} + placeholder={`${placeholder}（输入生产知识库 ID）`} + aria-label={placeholder} + /> + ) + + return ( +
+
+
+

生产知识库绑定

+

+ 这里有两件不同的事，页面分开显示：整库发布目标（封存/同步/恢复真正写入的目标， + 由部署配置决定，只读）与文档级绑定（旧流程：单篇文档发布写哪个库，可在这里改）。 +

+
+ +
+ {error ? ( +

+ {error} +

+ ) : null} + {bindings ? ( +
+

整库发布目标（实际写入，只读）

+ {whole?.configured === false ? ( +

+ 整库目标配置不全：{whole.missing_reason}。整库封存/同步/恢复在补齐部署配置前会被拒绝， + 页面不会用下面的文档级绑定顶上。 +

+ ) : null} + {wholeRows.length ? ( +
+ {wholeRows.map(([label, value]) => ( +
+
{label}
+
+ {value} +
+
+ ))} +
+ ) : null} +

+ 这份目标来自部署环境，不在页面上编辑；改它需要改部署配置并重启 api/worker，不会由本页保存。 +

+
+ ) : null} + {bindings ? ( +
+

{bindings.legacy_scope || '文档级绑定（旧路径）'}

+

仅文档级发布读取这组绑定；整库发布（封存/同步/恢复）不读它。

+
+ {renderSide('对内文档级目标', bindings.internal_kb_id, bindings.internal_kb_name)} + {renderSide('对外文档级目标', bindings.external_kb_id, bindings.external_kb_name)} +
+
+ ) : null} + {bindings && !bindings.production_connected ? ( +

+ 生产端核对不可用：{bindings.production_message}。 + {bindings.production_base_url ? `（生产地址：${bindings.production_base_url}）` : ''} +

+ ) : null} + {bindings && bindings.production_connected && bindings.production_base_url ? ( +

+ 生产端：{bindings.production_base_url} + {bindings.updated_by ? ` · 最近修改：${bindings.updated_by}` : ''} +

+ ) : null} + {isAdmin && bindings ? ( +
+ + +

保存时只改文档级绑定；整库发布目标由部署配置决定，本页不改它。

+

保存时会向生产端核对所选的库是否真实存在；对不上会拒收，生产端不可达则只提示。

+ {sameSelection ?

对内与对外必须绑定不同的生产知识库

: null} + + {message ? ( +

+ {message} +

+ ) : null} +
+ ) : null} + {!isAdmin ?

仅管理员可修改文档级发布目标。

: null} +
+ ) +} + +// 容器：读取 + 保存。整库目标只读展示，保存动作只写文档级绑定。 export function ProductionBindingsSection() { const auth = useOptionalAuth() const isAdmin = auth?.identity?.role === 'admin' @@ -56,8 +230,8 @@ export function ProductionBindingsSection() { const where = `对内 ${next.internal_kb_name || next.internal_kb_id.slice(0, 8) || '未绑定'} · 对外 ${next.external_kb_name || next.external_kb_id.slice(0, 8) || '未绑定'}` setMessage( next.production_connected - ? `发布目标已更新：${where}。之后新发布的“对内/对外”内容会写入这两个生产库（已发布过的条目仍回到原目标）。` - : `发布目标已保存：${where}；但当前核对不了生产端（${next.production_message}），下次发布前请确认。` + ? `文档级发布目标已更新：${where}。只影响单篇文档发布；整库发布写的是上面那份「整库发布目标」。` + : `文档级发布目标已保存：${where}；但当前核对不了生产端（${next.production_message}），下次发布前请确认。` ) } catch (cause) { setError(cause instanceof Error ? cause.message : '保存生产知识库绑定失败') @@ -66,89 +240,23 @@ export function ProductionBindingsSection() { } } - const optionLabel = (base: KnowledgeBaseOption) => `${base.name}（${base.knowledge_count} 文档）` - const renderSide = (label: string, boundID: string, boundName: string) => ( -
- {label} - - {boundID ? boundName || `${boundID.slice(0, 8)}…` : '未绑定 · 按同 ID 语义发布（目标端须存在同 ID 库）'} - - {boundID ? 'UI 绑定 · 发布写入此库' : '未绑定：沿用测试库 ID'} -
- ) - const editorFor = (value: string, onChange: (next: string) => void, placeholder: string) => - options.length > 0 ? ( - - ) : ( - onChange(event.target.value.trim())} - placeholder={`${placeholder}（输入生产知识库 ID）`} - aria-label={placeholder} - /> - ) - return ( -
-
-
-

生产知识库绑定

-

- 发布链路的“源库 → 目标库”：新发布的对内 / 对外内容按这里的绑定写入生产 WeKnora - 的对应知识库。留空表示未绑定，按同 ID 语义发布。 -

-
- -
- {error ? ( -

- {error} -

- ) : null} - {bindings ? ( -
- {renderSide('对内生产知识库', bindings.internal_kb_id, bindings.internal_kb_name)} - {renderSide('对外生产知识库', bindings.external_kb_id, bindings.external_kb_name)} -
- ) : null} - {bindings && !bindings.production_connected ? ( -

- 生产端核对不可用：{bindings.production_message}。 - {bindings.production_base_url ? `（生产地址：${bindings.production_base_url}）` : ''} -

- ) : null} - {bindings && bindings.production_connected && bindings.production_base_url ? ( -

- 生产端：{bindings.production_base_url} - {bindings.updated_by ? ` · 最近修改：${bindings.updated_by}` : ''} -

- ) : null} - {isAdmin && bindings ? ( -
- - -

保存时会向生产端核对所选的库是否真实存在；对不上会拒收，生产端不可达则只提示。

- {sameSelection ?

对内与对外必须绑定不同的生产知识库

: null} - - {message ? ( -

- {message} -

- ) : null} -
- ) : null} - {!isAdmin ?

仅管理员可修改发布目标。

: null} -
+ void save()} + onReload={() => void load()} + /> ) -} +} \ No newline at end of file diff --git a/web/src/ReleaseTargets.tsx b/web/src/ReleaseTargets.tsx index cd2adc4..443d69e 100644 --- a/web/src/ReleaseTargets.tsx +++ b/web/src/ReleaseTargets.tsx @@ -27,6 +27,7 @@ import { } from './librarySnapshots' import { useOptionalAuth } from './auth' import './releaseTargets.css' +import { RestoreVerificationSection } from './RestoreVerificationSection' // 发布管理：按已确认稿实现——顶部两个知识库页签，下面是**一个**作业区。 // @@ -260,6 +261,7 @@ export function jobKindText(kind?: string): string { if (kind === 'sync') return '生产同步' if (kind === 'restore') return '按版本恢复' if (kind === 'baseline') return '建立目标基线' + if (kind === 'verify') return '恢复验证' return kind ? kind : '—' } @@ -281,7 +283,9 @@ export function publishStateText(job?: LibraryJobRecord): string { ? '恢复进行中（目标在维护中）' : job.kind === 'baseline' ? '建立基线进行中（只读捕获目标，不写内容）' - : '同步进行中（目标在维护中）' + : job.kind === 'verify' + ? '恢复验证进行中（目标在维护中）' + : '同步进行中（目标在维护中）' } if (job.state === 'succeeded') return `${jobKindText(job.kind)}已完成 · ${stamp(job.finished_at ?? job.updated_at)}` if (job.state === 'failed') return `${jobKindText(job.kind)}失败 · ${stamp(job.finished_at ?? job.updated_at)}` @@ -718,6 +722,7 @@ function ProtectionCard({ targetSealed, sync, baseline, + scope, onOpen, }: { production: LibraryOverview['production'] | null @@ -730,6 +735,7 @@ function ProtectionCard({ targetSealed: boolean sync: ReleaseSyncActions baseline: ReleaseBaselineActions + scope: Scope onOpen: () => void }) { const backupKnown = prebackup.state === 'known' && prebackup.version @@ -798,6 +804,7 @@ function ProtectionCard({
恢复条件
{gate.reasons.length === 0 ? '已满足' : `缺 ${gate.reasons.length} 项`}
+ {gate.reasons.length > 0 ? (
    {gate.reasons.map((reason) => ( @@ -880,6 +887,7 @@ function ReadyBody({ targetSealed={Boolean(snapshot)} sync={sync} baseline={baseline} + scope={view.scope} onOpen={onOpen} /> diff --git a/web/src/RestoreVerificationSection.test.tsx b/web/src/RestoreVerificationSection.test.tsx new file mode 100644 index 0000000..b16d991 --- /dev/null +++ b/web/src/RestoreVerificationSection.test.tsx @@ -0,0 +1,103 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe, expect, it } from 'vitest' + +import { RestoreVerificationView } from './RestoreVerificationSection' +import type { LibraryVerificationView } from './restoreVerification' + +const target = { + instance: 'weknora-test', + project: 'weknora', + internal_kb: 'a8e9c8ad-411c-4c53-8421-599d29e9b179', + external_kb: '2e980686-2233-4c4e-b5a6-8b0069477b8e', + index: 'weknora_test_chunks', + s3_bucket: 'dobot-weknora-test', + tenant_id: 10000, + database_configured: true, +} + +const view = (overrides: Partial = {}): LibraryVerificationView => ({ + scope: 'internal', + target, + verified: false, + reason: '还没有对当前目标做过恢复验证', + latest: null, + history: [], + ...overrides, +}) + +const render = (value: LibraryVerificationView | null, isAdmin = true, pending = false, error: string | null = null) => + renderToStaticMarkup( + undefined} + onReload={() => undefined} + /> + ) + +describe('RestoreVerificationView', () => { + it('没有证据时显示"还没有验证记录"并给出下一步，不显示已验证', () => { + const html = render(view()) + expect(html).toContain('还没有验证记录') + expect(html).toContain('跑一次恢复验证') + expect(html).not.toContain('已验证（当前目标）') + }) + + it('管理员能看到发起按钮，工程师只看到证据与原因', () => { + expect(render(view(), true)).toContain('跑一次恢复验证') + const engineer = render(view(), false) + expect(engineer).not.toContain('跑一次恢复验证') + expect(engineer).toContain('仅管理员可发起') + }) + + it('通过后显示已验证、快照与完整目标', () => { + const html = render( + view({ + verified: true, + reason: '', + latest: { + id: 'libverify-internal-job1', + scope: 'internal', + status: 'passed', + snapshot_id: '0f9b39a16738', + job_id: 'libverify-internal-1', + started_at: '2026-11-03T01:00:00Z', + finished_at: '2026-11-03T01:02:00Z', + }, + }) + ) + expect(html).toContain('已验证（当前目标）') + expect(html).toContain('0f9b39a16738') + expect(html).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179') + expect(html).toContain('weknora_test_chunks') + }) + + it('失败时把失败原因写出来，按钮仍可重试', () => { + const html = render( + view({ + latest: { + id: 'libverify-internal-job2', + scope: 'internal', + status: 'failed', + snapshot_id: 'snap-2', + job_id: 'libverify-internal-2', + failure: '恢复后内容仍与预备份不一致', + started_at: '2026-11-03T01:00:00Z', + finished_at: '2026-11-03T01:05:00Z', + }, + }) + ) + expect(html).toContain('最近一次未通过') + expect(html).toContain('恢复后内容仍与预备份不一致') + expect(html).toContain('跑一次恢复验证') + }) + + it('投递被拒时显示后端逐条原因', () => { + const html = render(view(), true, false, '恢复验证被拒绝：生产写入闸口关闭') + expect(html).toContain('生产写入闸口关闭') + }) +}) diff --git a/web/src/RestoreVerificationSection.tsx b/web/src/RestoreVerificationSection.tsx new file mode 100644 index 0000000..85d8233 --- /dev/null +++ b/web/src/RestoreVerificationSection.tsx @@ -0,0 +1,140 @@ +import { useCallback, useEffect, useRef, useState } from 'react' + +import { + enqueueLibraryVerification, + fetchLibraryVerification, + verificationStatusText, + verificationTargetLines, +} from './restoreVerification' +import type { LibraryVerificationView } from './restoreVerification' + +// 「恢复验证」面板：同步闸口认的"生产可恢复"证据只能由这里发起的一次真实演练产生。 +// +// 为什么要有这个入口：以前只有 CLI（现为 internal/libraryverify 的薄封装）能跑演练， +// 页面只能看到一个 env 开关。运维在页面上看到"缺 1 项"，却没有任何可点的下一步。 +// 这里把"跑一次真实演练"变成正常流程：它会停写入者、写 canary、按预备份恢复， +// 全过程由 worker 执行并登记真实终态；页面只显示结果，不提供手填开关。 + +export interface RestoreVerificationViewProps { + view: LibraryVerificationView | null + scope: string + isAdmin: boolean + pending: boolean + error: string | null + notice: string + onRun: () => void + onReload: () => void +} + +const stampText = (value: string): string => { + if (!value) return '—' + const parsed = new Date(value) + return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString() +} + +export function RestoreVerificationView(props: RestoreVerificationViewProps) { + const { view, scope, isAdmin, pending, error, notice, onRun, onReload } = props + const status = verificationStatusText(view) + const latest = view?.latest ?? null + const running = Boolean(view?.running) + const tone = view?.verified ? 'green' : latest?.status === 'failed' ? 'red' : 'gray' + return ( +
    +
    +

    恢复验证（当前整库目标）

    + {status} +
    + {view ? ( +
      + {verificationTargetLines(view.target, scope).map((line) => ( +
    • {line}
      • + ))} +
    + ) : null} +

    {view?.reason || '还没有读到验证状态。'}

    + {latest ? ( +

    + 最近一次：{latest.status === 'passed' ? '通过' : '未通过'} · 快照 {latest.snapshot_id.slice(0, 12) || '—'} ·{' '} + {stampText(latest.finished_at)} · 作业 {latest.job_id} +

    + ) : null} + {latest?.failure ?

    未通过原因：{latest.failure}

    : null} + {error ?

    {error}

    : null} + {notice ?

    {notice}

    : null} +
    + {isAdmin ? ( + + ) : ( + 恢复验证仅管理员可发起；工程师可以看到证据与原因。 + )} + +
    +

    + 验证会停住目标写入者、写一个隔离 canary 证明校验有效、再按预备份恢复并逐类复核； + 通过后证据按当前目标登记，换库/换索引/换桶后自动失效。 +

    +
    + ) +} + +// 容器：挂载即读一次，之后每 5 秒刷新（验证作业在 worker 里跑，页面只读证据）。 +export function RestoreVerificationSection({ scope, isAdmin }: { scope: string; isAdmin: boolean }) { + const [view, setView] = useState(null) + const [pending, setPending] = useState(false) + const [error, setError] = useState(null) + const [notice, setNotice] = useState('') + const mounted = useRef(true) + + const reload = useCallback( + (signal?: AbortSignal) => { + fetchLibraryVerification(scope, signal) + .then((next) => { + if (mounted.current && next) setView(next) + }) + .catch(() => undefined) + }, + [scope] + ) + + useEffect(() => { + mounted.current = true + const controller = new AbortController() + reload(controller.signal) + const timer = setInterval(() => reload(controller.signal), 5000) + return () => { + mounted.current = false + controller.abort() + clearInterval(timer) + } + }, [reload]) + + const onRun = useCallback(() => { + setPending(true) + setError(null) + setNotice('') + enqueueLibraryVerification(scope) + .then((accepted) => { + setNotice(`已投递恢复验证作业 ${accepted.job_id}；进度在作业进度里可见，完成后这里显示证据。`) + reload() + }) + .catch((err: unknown) => setError(err instanceof Error ? err.message : '投递恢复验证失败')) + .finally(() => setPending(false)) + }, [scope, reload]) + + return ( + reload()} + /> + ) +} diff --git a/web/src/SystemSettings.test.tsx b/web/src/SystemSettings.test.tsx index d49a9ee..2d5a6b5 100644 --- a/web/src/SystemSettings.test.tsx +++ b/web/src/SystemSettings.test.tsx @@ -332,7 +332,8 @@ describe('SystemSettingsView', () => { // 发布目标（源库 → 目标库）是独立分区，与测试端收件库分开。 const target = view('production-bindings') expect(target).toContain('id="settings-production-bindings"') - expect(target).toContain('新发布的对内 / 对外内容按这里的绑定写入生产 WeKnora 的对应知识库') + expect(target).toContain('整库发布目标') + expect(target).toContain('文档级绑定') }) it('does not show healthy simulation when the settings data failed to load', () => { const html = renderToStaticMarkup( diff --git a/web/src/backupSchedule.ts b/web/src/backupSchedule.ts index 72b2b6d..b0a772f 100644 --- a/web/src/backupSchedule.ts +++ b/web/src/backupSchedule.ts @@ -12,6 +12,8 @@ export interface BackupSchedule { last_applied_summary: string host_configured: boolean host_target: string + // host_endpoint 是这条别名真实拨号的 host:port（来自挂载的 ssh 配置）。 + host_endpoint: string host_reachable: boolean host_error: string host_schedule_enabled: boolean @@ -70,6 +72,7 @@ function scheduleProjection(item: Record): BackupSchedule { last_applied_summary: asString(item.last_applied_summary), host_configured: item.host_configured === true, host_target: asString(item.host_target), + host_endpoint: asString(item.host_endpoint), host_reachable: item.host_reachable === true, host_error: asString(item.host_error), host_schedule_enabled: item.host_schedule_enabled === true, diff --git a/web/src/librarySnapshots.ts b/web/src/librarySnapshots.ts index 83c6446..54f478e 100644 --- a/web/src/librarySnapshots.ts +++ b/web/src/librarySnapshots.ts @@ -567,7 +567,7 @@ export interface LibraryStageRow { export interface LibraryJobRecord { id: string scope: string - kind: 'seal' | 'sync' | 'restore' | 'baseline' + kind: 'seal' | 'sync' | 'restore' | 'baseline' | 'verify' state: 'pending' | 'running' | 'succeeded' | 'failed' | 'cancelled' stage: string request_digest: string diff --git a/web/src/productionBindings.ts b/web/src/productionBindings.ts index 395ecb0..944e71c 100644 --- a/web/src/productionBindings.ts +++ b/web/src/productionBindings.ts @@ -11,6 +11,22 @@ export interface ProductionBindings { production_base_url: string production_connected: boolean production_message: string + // legacy_scope 标明这条记录属于旧文档级发布路径；整库发布不看它。 + legacy_scope: string + // whole_target 是整库发布真正写入的目标（来自部署配置），只读。 + whole_target: LibraryPublishTarget +} + +export interface LibraryPublishTarget { + configured: boolean + source: string + tenant_id: number + internal_kb_id: string + external_kb_id: string + database: string + elastic_index: string + object_bucket: string + missing_reason: string } const record = (value: unknown): value is Record => @@ -26,6 +42,24 @@ function projection(item: Record): ProductionBindings { production_base_url: typeof item.production_base_url === 'string' ? item.production_base_url : '', production_connected: item.production_connected === true, production_message: typeof item.production_message === 'string' ? item.production_message : '', + legacy_scope: typeof item.legacy_scope === 'string' ? item.legacy_scope : '', + whole_target: wholeTarget(item.whole_target), + } +} + +// wholeTarget 只读投影：缺字段就说未配置，不拿可编辑的文档级绑定冒充整库目标。 +function wholeTarget(value: unknown): LibraryPublishTarget { + const item = record(value) ? value : {} + return { + configured: item.configured === true, + source: typeof item.source === 'string' ? item.source : '', + tenant_id: typeof item.tenant_id === 'number' ? item.tenant_id : 0, + internal_kb_id: typeof item.internal_kb_id === 'string' ? item.internal_kb_id : '', + external_kb_id: typeof item.external_kb_id === 'string' ? item.external_kb_id : '', + database: typeof item.database === 'string' ? item.database : '', + elastic_index: typeof item.elastic_index === 'string' ? item.elastic_index : '', + object_bucket: typeof item.object_bucket === 'string' ? item.object_bucket : '', + missing_reason: typeof item.missing_reason === 'string' ? item.missing_reason : '', } } diff --git a/web/src/restoreVerification.test.ts b/web/src/restoreVerification.test.ts new file mode 100644 index 0000000..02f0689 --- /dev/null +++ b/web/src/restoreVerification.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest' + +import { + parseVerificationRecord, + parseVerificationView, + verificationStatusText, + verificationTargetLines, +} from './restoreVerification' +import type { LibraryVerificationView } from './restoreVerification' + +const target = { + instance: 'weknora-test', + project: 'weknora', + internal_kb: 'a8e9c8ad-411c-4c53-8421-599d29e9b179', + external_kb: '2e980686-2233-4c4e-b5a6-8b0069477b8e', + index: 'weknora_test_chunks', + s3_bucket: 'dobot-weknora-test', + tenant_id: 10000, + database_configured: true, +} + +const passed = { + id: 'libverify-internal-job1', + scope: 'internal', + status: 'passed', + snapshot_id: '0f9b39a16738', + job_id: 'libverify-internal-1', + started_at: '2026-11-03T01:00:00Z', + finished_at: '2026-11-03T01:02:00Z', +} + +describe('restoreVerification parsing', () => { + it('只把后端明确 true 当作已验证', () => { + const view = parseVerificationView({ scope: 'internal', target, verified: 'true', reason: '', history: [] }) + expect(view?.verified).toBe(false) + }) + + it('保留失败证据的失败原因', () => { + const record = parseVerificationRecord({ ...passed, status: 'failed', failure: '恢复后内容仍与预备份不一致' }) + expect(record?.status).toBe('failed') + expect(record?.failure).toBe('恢复后内容仍与预备份不一致') + }) + + it('未知状态一律丢弃，不猜成通过', () => { + expect(parseVerificationRecord({ ...passed, status: 'running' })).toBeNull() + }) + + it('没有目标信息的响应视为无效', () => { + expect(parseVerificationView({ scope: 'internal', verified: true })).toBeNull() + }) +}) + +describe('restoreVerification display', () => { + const base: LibraryVerificationView = { + scope: 'internal', + target, + verified: true, + reason: '', + latest: { + id: passed.id, + scope: 'internal', + status: 'passed', + snapshot_id: passed.snapshot_id, + job_id: passed.job_id, + started_at: passed.started_at, + finished_at: passed.finished_at, + }, + history: [], + } + + it('整库目标显示完整 KB ID 与索引/桶，不缩写成"2074 文档库"', () => { + const lines = verificationTargetLines(target, 'internal') + expect(lines.join(' | ')).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179') + expect(lines.join(' | ')).toContain('weknora_test_chunks') + expect(lines.join(' | ')).toContain('dobot-weknora-test') + }) + + it('对外 scope 显示对外目标', () => { + expect(verificationTargetLines(target, 'external').join(' | ')).toContain('2e980686-2233-4c4e-b5a6-8b0069477b8e') + }) + + it('状态文案区分已验证 / 最近失败 / 无记录', () => { + expect(verificationStatusText(base)).toBe('已验证（当前目标）') + expect( + verificationStatusText({ + ...base, + verified: false, + latest: { ...base.latest!, status: 'failed' }, + }) + ).toBe('最近一次未通过') + expect(verificationStatusText({ ...base, verified: false, latest: null })).toBe('还没有验证记录') + }) +}) diff --git a/web/src/restoreVerification.ts b/web/src/restoreVerification.ts new file mode 100644 index 0000000..3adc142 --- /dev/null +++ b/web/src/restoreVerification.ts @@ -0,0 +1,158 @@ +// 恢复验证的页面数据层：读证据 + 投递真实演练。 +// +// 关键约定：verified 只由后端证据回答（当前目标 + scope 的最近一次真实演练）。 +// 页面不提供任何"我确认已验证"的开关——那正是旧实现的问题（一个可手填的 env）。 +export type LibraryVerificationStatus = 'passed' | 'failed' + +export interface LibraryVerificationRecord { + id: string + scope: string + status: LibraryVerificationStatus + snapshot_id: string + job_id: string + failure?: string + started_at: string + finished_at: string +} + +export interface LibraryVerificationTarget { + instance: string + project: string + internal_kb: string + external_kb: string + index: string + s3_bucket: string + tenant_id: number + database_configured: boolean +} + +export interface LibraryVerificationView { + scope: string + target: LibraryVerificationTarget + verified: boolean + reason: string + latest?: LibraryVerificationRecord | null + history: LibraryVerificationRecord[] + running?: boolean + job_id?: string +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +function text(value: unknown): string { + return typeof value === 'string' ? value : '' +} + +function bool(value: unknown): boolean { + return value === true +} + +export function parseVerificationTarget(value: unknown): LibraryVerificationTarget | null { + if (!record(value)) return null + return { + instance: text(value.instance), + project: text(value.project), + internal_kb: text(value.internal_kb), + external_kb: text(value.external_kb), + index: text(value.index), + s3_bucket: text(value.s3_bucket), + tenant_id: typeof value.tenant_id === 'number' ? value.tenant_id : 0, + database_configured: bool(value.database_configured), + } +} + +export function parseVerificationRecord(value: unknown): LibraryVerificationRecord | null { + if (!record(value)) return null + const status = text(value.status) + if (status !== 'passed' && status !== 'failed') return null + return { + id: text(value.id), + scope: text(value.scope), + status, + snapshot_id: text(value.snapshot_id), + job_id: text(value.job_id), + failure: text(value.failure) || undefined, + started_at: text(value.started_at), + finished_at: text(value.finished_at), + } +} + +export function parseVerificationView(value: unknown): LibraryVerificationView | null { + if (!record(value)) return null + const target = parseVerificationTarget(value.target) + if (!target) return null + const history = Array.isArray(value.history) + ? value.history.map(parseVerificationRecord).filter((item): item is LibraryVerificationRecord => item !== null) + : [] + const latest = parseVerificationRecord(value.latest) ?? null + return { + scope: text(value.scope), + target, + // 只有后端明确说 true 才算已验证：任何解析失败都退回"没有证据"。 + verified: bool(value.verified), + reason: text(value.reason), + latest, + history, + running: bool(value.running), + job_id: text(value.job_id) || undefined, + } +} + +export async function fetchLibraryVerification( + scope: string, + signal?: AbortSignal +): Promise { + const response = await fetch(`/api/v1/library/verifications?scope=${encodeURIComponent(scope)}`, { + headers: { Accept: 'application/json' }, + signal, + }) + const value = (await response.json().catch(() => null)) as unknown + return parseVerificationView(value) +} + +export interface LibraryVerificationAccepted { + job_id: string + scope: string +} + +// 投递一次真实恢复验证。失败时把后端的逐条拒绝原因原样显示， +// 不把它包装成"稍后再试"——否则用户不知道是闸口、维护还是目标不可达。 +export async function enqueueLibraryVerification(scope: string): Promise { + const response = await fetch('/api/v1/library/verifications', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ scope }), + }) + const value = (await response.json().catch(() => null)) as unknown + if (!response.ok) { + if (response.status === 403) throw new Error('恢复验证仅管理员可发起。') + if (record(value) && Array.isArray(value.reasons)) { + const reasons = value.reasons.map(text).filter(Boolean) + if (reasons.length > 0) throw new Error(`恢复验证被拒绝：${reasons.join('；')}`) + } + if (record(value) && text(value.error)) throw new Error(text(value.error)) + throw new Error(`投递恢复验证失败（HTTP ${response.status}）`) + } + if (!record(value) || !record(value.job)) throw new Error('恢复验证作业响应无效') + return { job_id: text(value.job.id), scope: text(value.job.scope) } +} + +// 目标展示：整库目标必须显示完整 ID 与端点，避免"看着像 2074 文档库"这类误读。 +export function verificationTargetLines(target: LibraryVerificationTarget, scope: string): string[] { + const kb = scope === 'internal' ? target.internal_kb : target.external_kb + return [ + `实例 ${target.instance || '未配置'} · 项目 ${target.project || '未配置'}`, + `知识库 ${kb || '未配置'}`, + `索引 ${target.index || '未配置'} · 桶 ${target.s3_bucket || '未配置'}`, + `租户 ${target.tenant_id || '未配置'} · 数据库${target.database_configured ? '已配置' : '未配置'}`, + ] +} + +export function verificationStatusText(view: LibraryVerificationView | null): string { + if (!view) return '还没有读到验证状态' + if (view.verified) return '已验证（当前目标）' + if (view.latest) return view.latest.status === 'failed' ? '最近一次未通过' : '未通过' + return '还没有验证记录' +}