根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关, 页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现, 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动: - 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口), cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize), 与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效, 同 target+scope+job 幂等)。 - worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写 library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功); 中断接管按同一作业 id 收掉维护窗口。 - API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整 整库目标、最近证据、失败原因),OpenAPI 同步。 - 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底, 证据存在时以证据为准。 - 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询),作业类型补 verify 文案。 测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定); 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
This commit is contained in:
@@ -1655,6 +1655,93 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/verifications": {
|
||||
"get": {
|
||||
"tags": [
|
||||
"Runtime"
|
||||
],
|
||||
"operationId": "getLibraryRestoreVerification",
|
||||
"summary": "当前整库目标的恢复验证证据:verified 只由真实演练记录回答(按 target_id 绑定,换目标自动失效),并返回完整目标与失败原因",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "scope",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"internal",
|
||||
"external"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "目标与证据:verified/reason/latest/history/target"
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalError"
|
||||
},
|
||||
"503": {
|
||||
"description": "恢复验证存储不可用或证据读取失败"
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"tags": [
|
||||
"Runtime"
|
||||
],
|
||||
"operationId": "startLibraryRestoreVerification",
|
||||
"summary": "发起一次真实恢复验证(管理员):维护隔离下捕获目标 → 校验基线 → 写 canary 证明校验有效 → 按预备份恢复 → 复核,通过后才产生同步闸口认的证据",
|
||||
"responses": {
|
||||
"202": {
|
||||
"description": "已创建 verify 作业(worker 在维护窗口内真实执行)"
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest"
|
||||
},
|
||||
"403": {
|
||||
"description": "不是管理员"
|
||||
},
|
||||
"409": {
|
||||
"description": "目标正在维护或已有在飞写作业(同步/恢复/基线/验证互斥)"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalError"
|
||||
},
|
||||
"503": {
|
||||
"description": "生产写入闸口关闭或恢复验证存储不可用"
|
||||
}
|
||||
},
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"scope"
|
||||
],
|
||||
"properties": {
|
||||
"scope": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"internal",
|
||||
"external"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/snapshots/{id}": {
|
||||
"get": {
|
||||
"tags": [
|
||||
|
||||
@@ -232,6 +232,8 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
LibraryProductionInternalKB: config.LibraryProductionInternalKB,
|
||||
LibraryProductionExternalKB: config.LibraryProductionExternalKB,
|
||||
LibraryProductionTenantID: config.LibraryProductionTenantID,
|
||||
LibraryProductionIndex: config.LibraryProductionIndex,
|
||||
LibraryProductionS3Bucket: config.LibraryProductionS3Bucket,
|
||||
LibraryRetentionVersions: config.LibraryRetentionVersions,
|
||||
LibraryRetentionDays: config.LibraryRetentionDays,
|
||||
BackupHost: backuphost.New(backuphost.Config{
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error {
|
||||
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
|
||||
if err := ledger.Seal(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := mark(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return write(ctx)
|
||||
}
|
||||
+12
-203
@@ -1,4 +1,4 @@
|
||||
// command librarydrill 在目标实例上做一次**真实**的恢复演练:
|
||||
// command librarydrill 是恢复演练的**命令行入口**,实现在 internal/libraryverify:
|
||||
//
|
||||
// 维护隔离(关入口/停写入者/证明排空)
|
||||
// → 捕获目标整库(真预备份,落归档)
|
||||
@@ -8,16 +8,14 @@
|
||||
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
|
||||
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
|
||||
//
|
||||
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境
|
||||
// (与 worker 同一份 LIBRARY_* 配置),不进命令行、不打印。
|
||||
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
|
||||
// 为什么只剩薄封装:页面要能点一次"恢复验证",worker 必须在进程内跑同一份实现。
|
||||
// 两份实现意味着"CLI 通过、页面失败"或反过来,演练结论就不可信了。
|
||||
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
@@ -26,35 +24,12 @@ import (
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/libraryfreeze"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarystack"
|
||||
"knowledge-governance/internal/libraryverify"
|
||||
)
|
||||
|
||||
type drillVerification struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Verified bool `json:"verified"`
|
||||
Error string `json:"error,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
type drillReport struct {
|
||||
Status string `json:"status"` // passed | failed | maintenance_held
|
||||
Scope string `json:"scope"`
|
||||
Target string `json:"target"`
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Project string `json:"project"`
|
||||
Counts map[string]int64 `json:"counts"`
|
||||
BaselineCheck *drillVerification `json:"baseline_check"`
|
||||
DriftCheck *drillVerification `json:"drift_check"` // 必须失败:证明确实能发现差异
|
||||
RestoreCheck *drillVerification `json:"restore_check"`
|
||||
Canary string `json:"canary"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
Stage string `json:"stage,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
Until map[string]string `json:"until,omitempty"`
|
||||
}
|
||||
// 报告类型直接复用共享实现:命令行与页面/worker 的证据字段完全一致。
|
||||
type drillVerification = libraryverify.Verification
|
||||
type drillReport = libraryverify.Report
|
||||
|
||||
func main() {
|
||||
scope := flag.String("scope", "internal", "internal|external")
|
||||
@@ -88,18 +63,7 @@ func recoverWindows() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
manager, err := librarystack.OpenMaintenance(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer manager.Close()
|
||||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
if err := coordinator.Recover(context.Background()); err != nil {
|
||||
if err := libraryverify.RecoverWindows(config); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("maintenance windows recovered; target should be back to its original service state")
|
||||
@@ -107,170 +71,15 @@ func recoverWindows() error {
|
||||
}
|
||||
|
||||
func run(scope, jobID string) (*drillReport, error) {
|
||||
if scope != "internal" && scope != "external" {
|
||||
return nil, errors.New("scope must be internal or external")
|
||||
}
|
||||
config, err := appconfig.Load(os.Getenv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
report := &drillReport{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
|
||||
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
|
||||
fail := func(stage string, cause error, held bool) (*drillReport, error) {
|
||||
report.Stage = stage
|
||||
report.Failure = cause.Error()
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
if held {
|
||||
report.Status = "maintenance_held"
|
||||
}
|
||||
return report, cause
|
||||
}
|
||||
|
||||
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
|
||||
if err != nil {
|
||||
return fail("open_target", err, false)
|
||||
}
|
||||
defer target.Close()
|
||||
manager, err := librarystack.OpenMaintenance(config)
|
||||
if err != nil {
|
||||
return fail("open_maintenance", err, false)
|
||||
}
|
||||
defer manager.Close()
|
||||
if err := manager.VerifyTopology(ctx); err != nil {
|
||||
return fail("topology", err, false)
|
||||
}
|
||||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||||
if err != nil {
|
||||
return fail("open_coordinator", err, false)
|
||||
}
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
|
||||
if jobID == "" {
|
||||
jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405")
|
||||
jobID = fmt.Sprintf("drill-%s-%d", scope, time.Now().UTC().UnixNano())
|
||||
}
|
||||
var (
|
||||
snapshotID string
|
||||
counts map[string]int64
|
||||
canary string
|
||||
contentTouched bool
|
||||
baseline drillVerification
|
||||
drift drillVerification
|
||||
restored drillVerification
|
||||
)
|
||||
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
||||
start := time.Now()
|
||||
current, err := target.Current(ctx)
|
||||
if err != nil {
|
||||
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
|
||||
}
|
||||
counts = current.Counts
|
||||
report.Counts = counts
|
||||
// 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。
|
||||
backup, err := backups.Prepare(ctx, "", current)
|
||||
if err != nil {
|
||||
return "", errors.New("捕获目标整库失败:" + err.Error())
|
||||
}
|
||||
snapshotID = backup.SnapshotID
|
||||
report.SnapshotID = snapshotID
|
||||
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
|
||||
return "", errors.New("记录预备份身份失败:" + err.Error())
|
||||
}
|
||||
log.Printf("drill: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
|
||||
|
||||
// 2) 基线校验:预备份必须与目标完全一致。
|
||||
baseline = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||||
baseline.Error = err.Error()
|
||||
report.BaselineCheck = &baseline
|
||||
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
|
||||
}
|
||||
baseline.Verified = true
|
||||
report.BaselineCheck = &baseline
|
||||
|
||||
// Canary is an isolated tag, including on an empty target. Never seed
|
||||
// a document or change an existing row merely to make a drill possible.
|
||||
var random [16]byte
|
||||
if _, err := rand.Read(random[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
canary = fmt.Sprintf("%x", random[:])
|
||||
identityValue := target.Identity()
|
||||
var exists bool
|
||||
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if exists {
|
||||
return "", errors.New("canary identity collision; no write attempted")
|
||||
}
|
||||
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
|
||||
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
|
||||
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
|
||||
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
|
||||
report.Canary = canary
|
||||
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
|
||||
func(ctx context.Context) error {
|
||||
if err := window.MarkContentRisk(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
contentTouched = true
|
||||
return nil
|
||||
}, func(ctx context.Context) error {
|
||||
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
|
||||
return err
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("canary preparation/write failed: %w", err)
|
||||
}
|
||||
drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
||||
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
|
||||
report.DriftCheck = &drift
|
||||
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
|
||||
} else {
|
||||
drift.Error = verifyErr.Error()
|
||||
}
|
||||
report.DriftCheck = &drift
|
||||
log.Printf("drill: drift detected as expected (%s)", drift.Error)
|
||||
|
||||
// 4) 按预备份恢复。
|
||||
// Restore removes only the exact new canary identity in the durable write set.
|
||||
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
|
||||
WriteSet: &ledger}); err != nil {
|
||||
return "", errors.New("按预备份恢复失败:" + err.Error())
|
||||
}
|
||||
// 5) 恢复后逐类复核:必须与预备份完全一致。
|
||||
restored = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||||
restored.Error = err.Error()
|
||||
report.RestoreCheck = &restored
|
||||
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
|
||||
}
|
||||
restored.Verified = true
|
||||
report.RestoreCheck = &restored
|
||||
log.Printf("drill: restored and verified snapshot=%s", snapshotID)
|
||||
return snapshotID, nil
|
||||
}
|
||||
|
||||
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
||||
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
||||
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
|
||||
func(error) bool { return !contentTouched || restored.Verified })
|
||||
|
||||
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
||||
if windowErr != nil || window.State != "succeeded" {
|
||||
cause := windowErr
|
||||
if cause == nil {
|
||||
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
|
||||
}
|
||||
if window.State == "held" {
|
||||
return fail("window", cause, true)
|
||||
}
|
||||
return fail("window", cause, false)
|
||||
}
|
||||
report.Status = "passed"
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report, nil
|
||||
return libraryverify.Run(context.TODO(), config, scope, jobID,
|
||||
func(stage, state, detail string) { log.Printf("stage=%s state=%s %s", stage, state, detail) })
|
||||
}
|
||||
|
||||
func writeReport(path string, report *drillReport) error {
|
||||
|
||||
@@ -233,8 +233,8 @@ func runLibraryReleaseAttempt(ctx context.Context, writer *libraryAttemptWriter,
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
gateReasons = append(gateReasons, "生产写入闸口关闭")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(ctx, store, config, payload.Scope); !ok {
|
||||
gateReasons = append(gateReasons, reason)
|
||||
}
|
||||
if len(gateReasons) > 0 {
|
||||
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
|
||||
@@ -628,6 +628,10 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore,
|
||||
// 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口,
|
||||
// 继续捕获/校验,而不是留下一个 held 目标等人工处理。
|
||||
queueType = libraryBaselineJobType
|
||||
case governance.LibraryJobVerify:
|
||||
// 恢复验证同样持有维护窗口:接管时必须用同一份作业 id 收掉那个窗口,
|
||||
// 否则目标会一直停在维护里而作业显示失败。
|
||||
queueType = libraryVerifyJobType
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
|
||||
"target_snapshot_id": job.TargetSnapshotID})
|
||||
|
||||
@@ -30,4 +30,9 @@ func registerLibraryReleaseHandlers(handlers map[string]jobs.Handler, store *pos
|
||||
handlers[libraryBaselineJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
|
||||
return runLibraryBaseline(ctx, store, config, job)
|
||||
})
|
||||
// 恢复验证:真实"备份→canary→恢复→复核",通过后写入与当前目标绑定的可恢复证据。
|
||||
// 它是同步闸口的证据来源,所以与上面三类共用同一把执行锁与准入。
|
||||
handlers[libraryVerifyJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
|
||||
return runLibraryVerify(ctx, store, config, job)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -15,7 +15,8 @@ import (
|
||||
func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
|
||||
handlers := map[string]jobs.Handler{}
|
||||
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, nil)
|
||||
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType}
|
||||
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
|
||||
libraryBaselineJobType, libraryVerifyJobType}
|
||||
if len(handlers) != len(want) {
|
||||
t.Fatalf("registered %d handlers, want %d: %#v", len(handlers), len(want), handlers)
|
||||
}
|
||||
@@ -30,7 +31,8 @@ func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
|
||||
func TestRegisterLibraryReleaseHandlersFailsClosedWhenNotAdmitted(t *testing.T) {
|
||||
handlers := map[string]jobs.Handler{}
|
||||
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, errNotAdmitted{})
|
||||
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType} {
|
||||
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
|
||||
libraryBaselineJobType, libraryVerifyJobType} {
|
||||
err := handlers[jobType](context.Background(), jobs.Job{})
|
||||
if err == nil || !strings.Contains(err.Error(), "准入未通过") {
|
||||
t.Fatalf("handler %q error = %v, want admission refusal", jobType, err)
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/libraryverify"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
// libraryVerifyPayload 是"恢复验证"的队列载荷。
|
||||
//
|
||||
// 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方
|
||||
// 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。
|
||||
type libraryVerifyPayload struct {
|
||||
LibraryJobID string `json:"library_job_id"`
|
||||
Scope string `json:"scope"`
|
||||
}
|
||||
|
||||
// runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。
|
||||
//
|
||||
// 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary
|
||||
// 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。
|
||||
// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。
|
||||
func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||||
var payload libraryVerifyPayload
|
||||
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
|
||||
(payload.Scope != "internal" && payload.Scope != "external") {
|
||||
return errors.New("invalid library verify payload")
|
||||
}
|
||||
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope,
|
||||
governance.LibraryJobVerify, true); admitErr != nil {
|
||||
return admitErr
|
||||
}
|
||||
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID,
|
||||
func(runCtx context.Context, writer *libraryAttemptWriter) error {
|
||||
return runLibraryVerifyAttempt(runCtx, writer, config, payload)
|
||||
})
|
||||
}
|
||||
|
||||
func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
|
||||
payload libraryVerifyPayload) error {
|
||||
store := writer
|
||||
now := time.Now().UTC()
|
||||
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
|
||||
operationID: writer.attempt.OperationID}
|
||||
|
||||
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
|
||||
if jobErr != nil {
|
||||
// 作业行读不到就没有可登记的终态:直接失败,不落任何证据。
|
||||
return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr)
|
||||
}
|
||||
if existing.State == governance.LibraryJobSucceeded {
|
||||
// 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。
|
||||
recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行")
|
||||
return nil
|
||||
}
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行",
|
||||
"capture")
|
||||
}
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture",
|
||||
"", "", "", "", now)
|
||||
|
||||
report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID,
|
||||
func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) })
|
||||
|
||||
// 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。
|
||||
evidence := governance.LibraryRestoreVerification{
|
||||
ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID,
|
||||
Scope: payload.Scope,
|
||||
JobID: payload.LibraryJobID,
|
||||
Status: governance.LibraryVerificationFailed,
|
||||
StartedAt: now,
|
||||
}
|
||||
if report != nil {
|
||||
evidence.StartedAt = report.StartedAt
|
||||
evidence.SnapshotID = report.SnapshotID
|
||||
evidence.Report = verificationReportObject(report)
|
||||
}
|
||||
if runErr == nil && report != nil && report.Status == "passed" {
|
||||
evidence.Status = governance.LibraryVerificationPassed
|
||||
evidence.FinishedAt = report.FinishedAt
|
||||
} else {
|
||||
evidence.FinishedAt = time.Now().UTC()
|
||||
if report != nil && report.FinishedAt.After(evidence.StartedAt) {
|
||||
evidence.FinishedAt = report.FinishedAt
|
||||
}
|
||||
switch {
|
||||
case runErr != nil:
|
||||
evidence.Failure = runErr.Error()
|
||||
case report != nil && report.Status != "passed":
|
||||
evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")"
|
||||
default:
|
||||
evidence.Failure = "恢复验证未产生报告"
|
||||
}
|
||||
}
|
||||
if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil {
|
||||
// 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize")
|
||||
}
|
||||
|
||||
if runErr != nil || evidence.Status != governance.LibraryVerificationPassed {
|
||||
held := report != nil && report.Status == "maintenance_held"
|
||||
message := "恢复验证未通过:" + evidence.Failure
|
||||
if held {
|
||||
message += "(目标仍处于维护窗口,需要先按预备份恢复再重试)"
|
||||
}
|
||||
recovery := "未对目标执行任何写入,无需恢复"
|
||||
if report != nil && report.SnapshotID != "" {
|
||||
recovery = "目标已按预备份 " + report.SnapshotID + " 恢复并复核;下次验证会重新捕获"
|
||||
}
|
||||
snapshot := ""
|
||||
if report != nil {
|
||||
snapshot = report.SnapshotID
|
||||
}
|
||||
recorder.Stage(ctx, "finalize", "failed", message)
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
|
||||
message, recovery, snapshot, snapshot, time.Now().UTC())
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
finishedAt := evidence.FinishedAt
|
||||
if finishedAt.IsZero() {
|
||||
finishedAt = time.Now().UTC()
|
||||
}
|
||||
recorder.Stage(ctx, "finalize", "succeeded",
|
||||
"恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记")
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize",
|
||||
"", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt)
|
||||
log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。
|
||||
func verificationReportObject(report *libraryverify.Report) map[string]any {
|
||||
if report == nil {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
return map[string]any{"encode_error": err.Error()}
|
||||
}
|
||||
var object map[string]any
|
||||
if err := json.Unmarshal(raw, &object); err != nil {
|
||||
return map[string]any{"encode_error": err.Error()}
|
||||
}
|
||||
return object
|
||||
}
|
||||
|
||||
// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。
|
||||
//
|
||||
// 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶
|
||||
// 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED
|
||||
// 仍作为显式覆盖保留(不删除旧部署能力),但页面不再需要它,也不再由页面写入。
|
||||
func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface {
|
||||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
}, config appconfig.Config, scope string) (bool, string) {
|
||||
if store != nil {
|
||||
evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||||
switch {
|
||||
case err != nil:
|
||||
if config.ProductionBackupRestoreVerified {
|
||||
return true, ""
|
||||
}
|
||||
return false, "读取恢复验证证据失败:" + err.Error()
|
||||
case found:
|
||||
if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok {
|
||||
return true, ""
|
||||
} else {
|
||||
return false, reason
|
||||
}
|
||||
}
|
||||
}
|
||||
if config.ProductionBackupRestoreVerified {
|
||||
return true, ""
|
||||
}
|
||||
return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)"
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 同步闸口的"恢复可验证"判断必须以当前目标的证据为准:
|
||||
// 没有证据 ⇒ 拒绝并说清怎么补;有 passed 证据 ⇒ 放行;只有失败证据 ⇒ 拒绝并带失败原因。
|
||||
// 历史 env 覆盖只在证据缺失时兜底(页面不再写它)。
|
||||
type verifyEvidenceStore struct {
|
||||
evidence governance.LibraryRestoreVerification
|
||||
found bool
|
||||
}
|
||||
|
||||
func (s verifyEvidenceStore) LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
return s.evidence, s.found, nil
|
||||
}
|
||||
|
||||
func TestLibraryRestoreEvidenceSatisfied(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
passed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
|
||||
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1", FinishedAt: now}}
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), passed, appconfig.Config{}, "internal"); !ok {
|
||||
t.Fatalf("passed evidence must satisfy gate: %q", reason)
|
||||
}
|
||||
failed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
|
||||
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-1",
|
||||
Failure: "恢复后内容仍与预备份不一致", FinishedAt: now}}
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), failed, appconfig.Config{}, "internal"); ok ||
|
||||
!strings.Contains(reason, "恢复后内容仍与预备份不一致") {
|
||||
t.Fatalf("failed evidence must block with its own failure: ok=%v reason=%q", ok, reason)
|
||||
}
|
||||
missing := verifyEvidenceStore{}
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing, appconfig.Config{}, "internal"); ok ||
|
||||
!strings.Contains(reason, "恢复验证") {
|
||||
t.Fatalf("missing evidence must block and point at the remedy: ok=%v reason=%q", ok, reason)
|
||||
}
|
||||
// 历史部署的 env 覆盖:证据缺失时才生效,证据存在时不再起作用。
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing,
|
||||
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); !ok {
|
||||
t.Fatalf("legacy env override must still allow when no evidence: %q", reason)
|
||||
}
|
||||
if ok, _ := libraryRestoreEvidenceSatisfied(context.Background(), failed,
|
||||
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); ok {
|
||||
t.Fatal("evidence must win over the legacy env override")
|
||||
}
|
||||
}
|
||||
@@ -110,6 +110,10 @@ const libraryRestoreJobType = "library.restore"
|
||||
// libraryBaselineJobType 是"建立目标权威基线"的队列类型(与 API 侧常量同值)。
|
||||
const libraryBaselineJobType = "library.baseline"
|
||||
|
||||
// libraryVerifyJobType 是"恢复验证"的队列类型:真实备份→canary→恢复→复核,
|
||||
// 通过后才会产生同步闸口认的恢复证据。与 API 侧常量同值。
|
||||
const libraryVerifyJobType = "library.verify"
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
@@ -73,12 +73,13 @@ func (r LibraryWorkerRole) LibraryJobKinds() []string {
|
||||
switch r {
|
||||
case LibraryWorkerRoleReleaseExecutor:
|
||||
return []string{string(governance.LibraryJobSync), string(governance.LibraryJobRestore),
|
||||
string(governance.LibraryJobRecover), string(governance.LibraryJobBaseline)}
|
||||
string(governance.LibraryJobRecover), string(governance.LibraryJobBaseline),
|
||||
string(governance.LibraryJobVerify)}
|
||||
case LibraryWorkerRoleCapture:
|
||||
return []string{string(governance.LibraryJobSeal)}
|
||||
default:
|
||||
return []string{string(governance.LibraryJobSeal), string(governance.LibraryJobSync),
|
||||
string(governance.LibraryJobRestore), string(governance.LibraryJobRecover),
|
||||
string(governance.LibraryJobBaseline)}
|
||||
string(governance.LibraryJobBaseline), string(governance.LibraryJobVerify)}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,13 +63,14 @@ func TestLoadRejectsUnknownLibraryWorkerRole(t *testing.T) {
|
||||
// 每个角色收拾哪些 kind、谁拥有 DDL,都必须互斥且覆盖完整。
|
||||
func TestLibraryWorkerRoleOwnershipMaps(t *testing.T) {
|
||||
release := LibraryWorkerRoleReleaseExecutor
|
||||
if got := release.LibraryJobKinds(); len(got) != 4 || got[0] != "sync" || got[1] != "restore" || got[2] != "recover" || got[3] != "baseline" {
|
||||
if got := release.LibraryJobKinds(); len(got) != 5 || got[0] != "sync" || got[1] != "restore" ||
|
||||
got[2] != "recover" || got[3] != "baseline" || got[4] != "verify" {
|
||||
t.Fatalf("release-executor kinds = %v", got)
|
||||
}
|
||||
if got := LibraryWorkerRoleCapture.LibraryJobKinds(); len(got) != 1 || got[0] != "seal" {
|
||||
t.Fatalf("capture kinds = %v", got)
|
||||
}
|
||||
if got := LibraryWorkerRoleFull.LibraryJobKinds(); len(got) != 5 {
|
||||
if got := LibraryWorkerRoleFull.LibraryJobKinds(); len(got) != 6 {
|
||||
t.Fatalf("full kinds = %v", got)
|
||||
}
|
||||
for _, role := range []LibraryWorkerRole{LibraryWorkerRoleFull, LibraryWorkerRoleCapture} {
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -127,12 +129,81 @@ func (r *Runner) runRemote(ctx context.Context, timeout time.Duration, remote st
|
||||
if ctx.Err() != nil {
|
||||
return text, fmt.Errorf("ssh %s 超时(%s)", r.config.Target, timeout)
|
||||
}
|
||||
// 主机侧脚本会用退出码表达"参数非法/有库失败",把输出带回去便于页面展示。
|
||||
return text, fmt.Errorf("ssh %s 执行失败: %s", r.config.Target, firstLines(text, 6))
|
||||
// 连接层失败与主机侧命令失败必须分开说:前者连脚本都没跑,
|
||||
// 任何"脚本缺失/计划未下发"都是把未观测当成事实。
|
||||
return text, hostFailure(r.config.Target, text)
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// hostFailure 把 ssh 的非零退出分成两类:连不上,和连上了但远端命令失败。
|
||||
func hostFailure(target, text string) error {
|
||||
if connectionFailurePattern.MatchString(text) {
|
||||
return fmt.Errorf("目标主机 %s 不可达(%s);未执行任何主机侧脚本,脚本与计划状态未知",
|
||||
target, firstLines(text, 3))
|
||||
}
|
||||
// 主机侧脚本会用退出码表达"参数非法/有库失败",把输出带回去便于页面展示。
|
||||
return fmt.Errorf("已连上 %s,但主机侧命令失败: %s", target, firstLines(text, 6))
|
||||
}
|
||||
|
||||
// connectionFailurePattern 是 OpenSSH 客户端在"根本没建立连接"时打的原文。
|
||||
// 只有命中它才允许说"不可达";其余非零退出属于远端命令自己的失败。
|
||||
var connectionFailurePattern = regexp.MustCompile(`(?i)ssh: connect to host|Operation timed out|Connection timed out|Connection refused|Connection reset by peer|No route to host|Network is unreachable|Could not resolve hostname|Host key verification failed|Permission denied|Connection closed by`)
|
||||
|
||||
// Endpoint 读回挂载的 ssh 配置里本别名真实拨号的 host:port。
|
||||
// 页面必须能说清"现在连的是哪个地址"——只显示别名时,改错文件也看不出来。
|
||||
func (r *Runner) Endpoint() string {
|
||||
if r == nil || r.config.SSHConfig == "" || r.config.Target == "" {
|
||||
return ""
|
||||
}
|
||||
raw, err := os.ReadFile(r.config.SSHConfig)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
host, port := sshConfigEndpoint(string(raw), r.config.Target)
|
||||
if host == "" {
|
||||
return ""
|
||||
}
|
||||
if port == "" {
|
||||
port = "22"
|
||||
}
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
|
||||
// sshConfigEndpoint 解析 `Host <alias>` 段的 HostName/Port(首个匹配段生效)。
|
||||
// 只做"页面显示用"的最小解析,不替代 ssh 自己的配置语义。
|
||||
func sshConfigEndpoint(config, alias string) (string, string) {
|
||||
host, port := "", ""
|
||||
active := false
|
||||
for _, line := range strings.Split(config, "\n") {
|
||||
fields := strings.Fields(strings.TrimSpace(line))
|
||||
if len(fields) == 0 || strings.HasPrefix(fields[0], "#") {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(fields[0])
|
||||
if key == "host" {
|
||||
active = false
|
||||
for _, pattern := range fields[1:] {
|
||||
if pattern == alias {
|
||||
active = true
|
||||
break
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !active || len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
switch key {
|
||||
case "hostname":
|
||||
host = strings.Trim(fields[1], `"`)
|
||||
case "port":
|
||||
port = strings.Trim(fields[1], `"`)
|
||||
}
|
||||
}
|
||||
return host, port
|
||||
}
|
||||
|
||||
var scheduleAppliedPattern = regexp.MustCompile(
|
||||
`SCHEDULE_APPLIED enabled=(\d)(?: time=(\S+))?(?: days=(\d+))?(?: months=(\d+))?(?: next=(\S+))?(?: cron=(\S+))?`)
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ package backuphost
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -188,6 +190,41 @@ func TestRunnerSurfacesHostFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 现场原错:ssh: connect to host 120.76.217.51 port 22: Operation timed out。
|
||||
// 连不上时脚本根本没跑,只能说"不可达";断言配对的反面:不能说脚本缺失。
|
||||
func TestHostFailureSeparatesUnreachableFromRemoteCommand(t *testing.T) {
|
||||
unreachable := hostFailure("weknora-test", "ssh: connect to host 120.76.217.51 port 22: Operation timed out")
|
||||
if !strings.Contains(unreachable.Error(), "不可达") || !strings.Contains(unreachable.Error(), "Operation timed out") {
|
||||
t.Fatalf("连接超时应报不可达并带原错: %v", unreachable)
|
||||
}
|
||||
if strings.Contains(unreachable.Error(), "主机侧命令失败") || strings.Contains(unreachable.Error(), "脚本缺失") {
|
||||
t.Fatalf("连接失败不能报成远端命令/脚本问题: %v", unreachable)
|
||||
}
|
||||
remote := hostFailure("weknora-test", "bash: /opt/weknora-test/logical-backup.sh: No such file or directory")
|
||||
if !strings.Contains(remote.Error(), "已连上") || !strings.Contains(remote.Error(), "No such file") {
|
||||
t.Fatalf("已连接但脚本失败时应原样带回原因: %v", remote)
|
||||
}
|
||||
if strings.Contains(remote.Error(), "不可达") {
|
||||
t.Fatalf("已连上就不能说不可达: %v", remote)
|
||||
}
|
||||
}
|
||||
|
||||
// 页面要能说清"实际拨号的地址",否则改错 ssh_config 也看不出来。
|
||||
func TestRunnerEndpointReadsDialedHostPort(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "ssh_config")
|
||||
config := "Host other\n HostName 10.0.0.9\n Port 2222\n\nHost weknora-test\n HostName 100.85.111.121\n HostKeyAlias 120.76.217.51\n"
|
||||
if err := os.WriteFile(path, []byte(config), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := New(Config{SSHConfig: path, Target: "weknora-test", ScriptDir: "/opt/weknora-test"})
|
||||
if got := runner.Endpoint(); got != "100.85.111.121:22" {
|
||||
t.Fatalf("应报真实拨号地址(默认 22),得到 %q", got)
|
||||
}
|
||||
if got := New(Config{SSHConfig: filepath.Join(t.TempDir(), "missing"), Target: "weknora-test"}).Endpoint(); got != "" {
|
||||
t.Fatalf("配置文件不可读时应返回空,不猜地址,得到 %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunNowReadsBackLastRun(t *testing.T) {
|
||||
calls := 0
|
||||
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
|
||||
|
||||
@@ -44,6 +44,10 @@ const (
|
||||
// 读出来封存成快照并逐类校验,再登记成 production_effective。
|
||||
// 它不写目标内容、不清空目标、不猜版本——基线只能来自真实捕获与校验。
|
||||
LibraryJobBaseline LibraryJobKind = "baseline"
|
||||
// LibraryJobVerify 是"恢复验证":在维护窗口内捕获目标 → 校验基线一致 →
|
||||
// 写一个 canary 证明校验能发现差异 → 按预备份恢复 → 复核完全一致。
|
||||
// 它是唯一能产生"生产可恢复"证据的作业,页面不再接受手填开关。
|
||||
LibraryJobVerify LibraryJobKind = "verify"
|
||||
)
|
||||
|
||||
type LibraryJobState string
|
||||
@@ -173,6 +177,12 @@ func BaselineStages() []string {
|
||||
return []string{"capture", "verify", "finalize"}
|
||||
}
|
||||
|
||||
// VerifyStages 是恢复验证的阶段:捕获 → 基线校验 → 写 canary(证明校验有效)
|
||||
// → 按预备份恢复 → 恢复后复核。canary 与 restore 都是真实写入,失败必须可见。
|
||||
func VerifyStages() []string {
|
||||
return []string{"capture", "verify", "canary", "restore", "finalize"}
|
||||
}
|
||||
|
||||
// LibraryReleaseAuthorization 是生产发布授权:绑定目标、快照、参数摘要与有效期。
|
||||
type LibraryReleaseAuthorization struct {
|
||||
ID string `json:"id"`
|
||||
|
||||
@@ -147,7 +147,7 @@ func ValidateLibraryOperation(op LibraryOperation) error {
|
||||
return errors.New("library operation requires internal or external scope")
|
||||
}
|
||||
switch op.Kind {
|
||||
case LibraryJobSeal, LibraryJobSync, LibraryJobRestore, LibraryJobBaseline, LibraryJobRecover:
|
||||
case LibraryJobSeal, LibraryJobSync, LibraryJobRestore, LibraryJobBaseline, LibraryJobRecover, LibraryJobVerify:
|
||||
default:
|
||||
return fmt.Errorf("unsupported library operation kind %q", op.Kind)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LibraryRestoreVerification 是一次**真实**恢复验证(备份 → canary → 恢复 → 复核)的登记。
|
||||
//
|
||||
// 为什么不能继续用环境变量 PRODUCTION_BACKUP_RESTORE_VERIFIED:那是一个可以手填的开关,
|
||||
// 它既证明不了"有人真跑过恢复",也证明不了"跑的是当前目标"。这里的每条记录由 worker
|
||||
// 在真实演练结束后写入,绑定 target_id(实例+库+索引+桶)与 scope,失败也照实留痕。
|
||||
type LibraryRestoreVerification struct {
|
||||
ID string `json:"id"`
|
||||
Scope string `json:"scope"`
|
||||
Status string `json:"status"` // passed | failed
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
JobID string `json:"job_id"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
Report map[string]any `json:"report,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
}
|
||||
|
||||
const (
|
||||
LibraryVerificationPassed = "passed"
|
||||
LibraryVerificationFailed = "failed"
|
||||
)
|
||||
|
||||
// LibraryVerificationSatisfied 只回答一件事:这条证据能不能支撑"当前目标的恢复验证已通过"。
|
||||
// 通过的条件必须是全部成立——状态 passed、scope 相同、快照非空、完成时间有效。
|
||||
// target_id 的匹配由查询层(按当前 target_id 过滤)保证,这里不重复比较。
|
||||
func LibraryVerificationSatisfied(verification LibraryRestoreVerification, scope string) (bool, string) {
|
||||
if verification.Status != LibraryVerificationPassed {
|
||||
reason := "最近一次恢复验证没有通过"
|
||||
if detail := strings.TrimSpace(verification.Failure); detail != "" {
|
||||
reason += ":" + detail
|
||||
}
|
||||
return false, reason
|
||||
}
|
||||
if verification.Scope != scope {
|
||||
return false, "恢复验证证据属于另一个 scope(" + verification.Scope + ")"
|
||||
}
|
||||
if strings.TrimSpace(verification.SnapshotID) == "" {
|
||||
return false, "恢复验证证据没有绑定快照,不能作为备份可恢复的证明"
|
||||
}
|
||||
if verification.FinishedAt.IsZero() {
|
||||
return false, "恢复验证证据没有完成时间"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 证据能不能支撑"当前目标可恢复"只由这四个条件回答:
|
||||
// 状态 passed、scope 相同、绑定了快照、有完成时间。少任何一条都不算。
|
||||
func TestLibraryVerificationSatisfied(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
cases := []struct {
|
||||
name string
|
||||
verification LibraryRestoreVerification
|
||||
scope string
|
||||
want bool
|
||||
reason string
|
||||
}{
|
||||
{name: "passed with snapshot", scope: "internal", want: true,
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal",
|
||||
SnapshotID: "snap-1", FinishedAt: now}},
|
||||
{name: "failed carries failure", scope: "internal", want: false, reason: "canary 后校验仍通过",
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationFailed, Scope: "internal",
|
||||
SnapshotID: "snap-1", Failure: "canary 后校验仍通过", FinishedAt: now}},
|
||||
{name: "wrong scope", scope: "internal", want: false, reason: "另一个 scope",
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "external",
|
||||
SnapshotID: "snap-1", FinishedAt: now}},
|
||||
{name: "no snapshot", scope: "internal", want: false, reason: "没有绑定快照",
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal",
|
||||
FinishedAt: now}},
|
||||
{name: "no finish time", scope: "internal", want: false, reason: "没有完成时间",
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal",
|
||||
SnapshotID: "snap-1"}},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
ok, reason := LibraryVerificationSatisfied(testCase.verification, testCase.scope)
|
||||
if ok != testCase.want {
|
||||
t.Fatalf("satisfied=%v want=%v reason=%q", ok, testCase.want, reason)
|
||||
}
|
||||
if !testCase.want && testCase.reason != "" && !containsText(reason, testCase.reason) {
|
||||
t.Fatalf("reason=%q want contains %q", reason, testCase.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func containsText(haystack, needle string) bool {
|
||||
return len(haystack) >= len(needle) && (haystack == needle || indexOf(haystack, needle) >= 0)
|
||||
}
|
||||
|
||||
func indexOf(haystack, needle string) int {
|
||||
for i := 0; i+len(needle) <= len(haystack); i++ {
|
||||
if haystack[i:i+len(needle)] == needle {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -24,6 +24,9 @@ type BackupScheduleStore interface {
|
||||
type BackupHost interface {
|
||||
Available() bool
|
||||
Target() string
|
||||
// Endpoint 是挂载的 ssh 配置里该别名真实拨号的 host:port;页面据此证明
|
||||
// "连的是哪条通道",而不是只显示一个别名。
|
||||
Endpoint() string
|
||||
Show(context.Context) (backuphost.ShowResult, error)
|
||||
Apply(context.Context, bool, string, int, int) (backuphost.ApplyResult, error)
|
||||
RunNow(context.Context) (backuphost.RunResult, error)
|
||||
@@ -82,8 +85,11 @@ type backupScheduleView struct {
|
||||
// HostConfigured/HostReachable 区分"没接线"与"接线了但连不上"。
|
||||
HostConfigured bool `json:"host_configured"`
|
||||
HostTarget string `json:"host_target"`
|
||||
HostReachable bool `json:"host_reachable"`
|
||||
HostError string `json:"host_error"`
|
||||
// HostEndpoint 是这条别名真实拨号的 host:port(来自挂载的 ssh 配置)。
|
||||
// 连接失败时它就是"到底往哪连"的唯一证据,不能只给一个别名让人猜。
|
||||
HostEndpoint string `json:"host_endpoint"`
|
||||
HostReachable bool `json:"host_reachable"`
|
||||
HostError string `json:"host_error"`
|
||||
|
||||
HostScheduleEnabled bool `json:"host_schedule_enabled"`
|
||||
HostScheduleTime string `json:"host_schedule_time"`
|
||||
@@ -281,6 +287,7 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
|
||||
}
|
||||
view.HostConfigured = config.BackupHost.Available()
|
||||
view.HostTarget = config.BackupHost.Target()
|
||||
view.HostEndpoint = config.BackupHost.Endpoint()
|
||||
if !view.HostConfigured {
|
||||
view.InSync = false
|
||||
view.HostError = "目标主机凭据未挂载(BACKUP_HOST_SSH_CONFIG / BACKUP_HOST_TARGET)"
|
||||
@@ -293,6 +300,9 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
|
||||
if err != nil {
|
||||
view.HostReachable = false
|
||||
view.HostError = err.Error()
|
||||
if view.HostEndpoint != "" {
|
||||
view.HostError += "(拨号地址 " + view.HostEndpoint + ")"
|
||||
}
|
||||
view.InSync = false
|
||||
} else {
|
||||
view.HostReachable = true
|
||||
|
||||
@@ -46,6 +46,7 @@ func (store *fakeBackupScheduleStore) MarkBackupScheduleApplied(_ context.Contex
|
||||
type fakeBackupHost struct {
|
||||
available bool
|
||||
target string
|
||||
endpoint string
|
||||
show backuphost.ShowResult
|
||||
showErr error
|
||||
apply backuphost.ApplyResult
|
||||
@@ -59,6 +60,9 @@ type fakeBackupHost struct {
|
||||
|
||||
func (host *fakeBackupHost) Available() bool { return host.available }
|
||||
func (host *fakeBackupHost) Target() string { return host.target }
|
||||
func (host *fakeBackupHost) Endpoint() string {
|
||||
return host.endpoint
|
||||
}
|
||||
func (host *fakeBackupHost) Show(context.Context) (backuphost.ShowResult, error) {
|
||||
return host.show, host.showErr
|
||||
}
|
||||
@@ -166,6 +170,48 @@ func TestHandleBackupScheduleHostFlagsDrift(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBackupScheduleHostUnreachableSeparatesConnectionFromHostState(t *testing.T) {
|
||||
// 现场原错:ssh connect to host 120.76.217.51 port 22 Operation timed out。
|
||||
// 连接没建立 ⇒ 既不能报"脚本缺失",也不能报"计划未下发",只能报不可达 + 拨号地址。
|
||||
host := &fakeBackupHost{
|
||||
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
|
||||
showErr: errors.New("目标主机 weknora-test 不可达(ssh: connect to host 120.76.217.51 port 22: Operation timed out);未执行任何主机侧脚本,脚本与计划状态未知"),
|
||||
}
|
||||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||||
recorder := httptest.NewRecorder()
|
||||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||||
data := decodeData(t, recorder)
|
||||
if data["host_reachable"] != false || data["in_sync"] != false {
|
||||
t.Fatalf("连不上时不能报告可达或已同步: %+v", data)
|
||||
}
|
||||
message := data["host_error"].(string)
|
||||
if !strings.Contains(message, "不可达") || !strings.Contains(message, "100.85.111.121:22") {
|
||||
t.Fatalf("应报不可达并给出真实拨号地址: %q", message)
|
||||
}
|
||||
if strings.Contains(message, "脚本缺失") || strings.Contains(message, "未下发") {
|
||||
t.Fatalf("未观测到的脚本/计划状态不能当成结论: %q", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBackupScheduleHostReportsMissingScriptOnlyAfterConnecting(t *testing.T) {
|
||||
// 真正的脚本缺失:ssh 已连上,主机回的是脚本不存在 ⇒ 这时才允许说脚本缺失。
|
||||
host := &fakeBackupHost{
|
||||
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
|
||||
showErr: errors.New("已连上 weknora-test,但主机侧命令失败: bash: /opt/weknora-test/install_backup_schedule.sh: No such file or directory"),
|
||||
}
|
||||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||||
recorder := httptest.NewRecorder()
|
||||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||||
data := decodeData(t, recorder)
|
||||
message := data["host_error"].(string)
|
||||
if !strings.Contains(message, "已连上") || !strings.Contains(message, "No such file") {
|
||||
t.Fatalf("已连接但脚本缺失时应原样带回主机原因: %q", message)
|
||||
}
|
||||
if strings.Contains(message, "不可达") {
|
||||
t.Fatalf("已连上就不能说不可达: %q", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBackupScheduleHostUnconfiguredDegrades(t *testing.T) {
|
||||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, &fakeBackupHost{available: false})
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
@@ -642,9 +642,7 @@ func librarySyncGateWithGrant(config Config, production libraryKnownView, prebac
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
gate.Reasons = append(gate.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
gate.Reasons = append(gate.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
|
||||
}
|
||||
gate.Reasons = append(gate.Reasons, restoreGateReasons(config)...)
|
||||
// 生产端点以"整库目标是否配置"为准:整库同步直接写 PG/ES/对象,
|
||||
// 不依赖 WeKnora 的 HTTP 端,所以不能拿它是否配置当门禁。
|
||||
if strings.TrimSpace(config.LibraryProductionDatabaseURL) == "" {
|
||||
@@ -814,7 +812,8 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi
|
||||
|
||||
grant, hasGrant := activeGrantFor(r.Context(), config, scope, target)
|
||||
overview.Authorization = authorizationViewOf(grant, hasGrant)
|
||||
overview.Gate = librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, target, hasGrant)
|
||||
// 闸口按当前目标的真实恢复证据判断(没有证据时也接受历史 env 覆盖)。
|
||||
overview.Gate = librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), scope), overview.Production, overview.Prebackup, target, hasGrant)
|
||||
writeJSON(w, http.StatusOK, overview)
|
||||
}
|
||||
|
||||
@@ -963,7 +962,7 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
return
|
||||
}
|
||||
}
|
||||
gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant)
|
||||
gate := librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), request.Scope), overview.Production, overview.Prebackup, request.Target, hasGrant)
|
||||
if !gate.SyncAllowed {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||||
"code": "library_sync_blocked", "reasons": gate.Reasons,
|
||||
|
||||
@@ -71,9 +71,7 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
reasons = append(reasons, "生产写入闸口关闭")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||||
}
|
||||
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), request.Scope))...)
|
||||
authorizationOK := false
|
||||
if storeWithAuth, ok := config.Store.(interface {
|
||||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||||
@@ -169,9 +167,7 @@ func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config C
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
view.Reasons = append(view.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
|
||||
}
|
||||
view.Reasons = append(view.Reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), scope))...)
|
||||
digest, err := libraryRestoreParamsDigest(scope, snapshotID)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||||
@@ -228,9 +224,7 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
reasons = append(reasons, "生产写入闸口关闭")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||||
}
|
||||
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), source.Scope))...)
|
||||
var digest string
|
||||
switch source.Kind {
|
||||
case governance.LibraryJobSync:
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 恢复验证:页面点一次"跑恢复验证",后端建一个持久作业,worker 用与 CLI 同一份实现
|
||||
// (internal/libraryverify)真实执行:捕获目标 → 基线校验 → 写 canary 证明校验有效 →
|
||||
// 按预备份恢复 → 恢复后复核。通过与否只由这条证据表回答,页面不接受手填开关。
|
||||
const libraryVerifyJobType = "library.verify"
|
||||
|
||||
type libraryVerifyRequest struct {
|
||||
Scope string `json:"scope"`
|
||||
}
|
||||
|
||||
// libraryVerificationView 是页面需要的全部事实:当前目标是谁、最近一次验证结果、
|
||||
// 证据是否对当前目标成立、以及同步闸口能不能据此放行。
|
||||
type libraryVerificationView struct {
|
||||
Scope string `json:"scope"`
|
||||
Target libraryVerificationTarget `json:"target"`
|
||||
Verified bool `json:"verified"`
|
||||
Reason string `json:"reason"`
|
||||
Latest *governance.LibraryRestoreVerification `json:"latest,omitempty"`
|
||||
History []governance.LibraryRestoreVerification `json:"history"`
|
||||
Running bool `json:"running"`
|
||||
JobID string `json:"job_id,omitempty"`
|
||||
}
|
||||
|
||||
type libraryVerificationTarget struct {
|
||||
Instance string `json:"instance"`
|
||||
Project string `json:"project"`
|
||||
InternalKB string `json:"internal_kb"`
|
||||
ExternalKB string `json:"external_kb"`
|
||||
Index string `json:"index"`
|
||||
S3Bucket string `json:"s3_bucket"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
DatabaseSet bool `json:"database_configured"`
|
||||
}
|
||||
|
||||
// libraryVerifyEnqueue 投递一次恢复验证作业(管理员)。
|
||||
//
|
||||
// 它不签发发布授权、不改版本:验证只回答"当前目标的预备份能不能恢复"。
|
||||
// 目标是部署配置,不接受请求体指定——否则一次点击就能验证别的库并把证据挂到当前目标上。
|
||||
func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||||
return
|
||||
}
|
||||
identity, ok := sessionFrom(r.Context())
|
||||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "恢复验证仅管理员可发起"})
|
||||
return
|
||||
}
|
||||
var request libraryVerifyRequest
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return
|
||||
}
|
||||
if request.Scope != "internal" && request.Scope != "external" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||||
return
|
||||
}
|
||||
store, ok := libraryStore(config)
|
||||
if !ok {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_verify_unavailable",
|
||||
"error": "恢复验证存储不可用"})
|
||||
return
|
||||
}
|
||||
// 验证要停写入者并在目标上写 canary:维护中或闸口关闭时直接拒绝,
|
||||
// 不让页面投一个 worker 必然失败的作业。
|
||||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||||
writeJSON(w, http.StatusConflict, body)
|
||||
return
|
||||
}
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||||
"code": "library_verify_blocked",
|
||||
"reasons": []string{
|
||||
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary",
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobVerify,
|
||||
map[string]any{"scope": request.Scope})
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"})
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
|
||||
ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
|
||||
Kind: governance.LibraryJobVerify,
|
||||
IdempotencyKey: "library.verify:" + request.Scope + ":" + now.Format("20060102T150405.000000000"),
|
||||
RequestDigest: digest, Actor: identity.Username,
|
||||
}, governance.VerifyStages(), false, now)
|
||||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active",
|
||||
"error": "该目标已有在飞的写作业(同步/恢复/基线/验证互斥),等它结束再发起"})
|
||||
return
|
||||
}
|
||||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
|
||||
"error": "同一幂等键对应了不同的请求"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify job"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": reused})
|
||||
}
|
||||
|
||||
// libraryVerifications 读当前目标的恢复验证证据(页面轮询用)。
|
||||
func libraryVerifications(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
scope, err := scopeFromQuery(r)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
view := libraryVerificationView{
|
||||
Scope: scope,
|
||||
History: []governance.LibraryRestoreVerification{},
|
||||
Target: libraryVerificationTarget{
|
||||
Instance: config.LibraryTargetInstance,
|
||||
Project: config.LibraryTargetProject,
|
||||
InternalKB: config.LibraryProductionInternalKB,
|
||||
ExternalKB: config.LibraryProductionExternalKB,
|
||||
Index: config.LibraryProductionIndex,
|
||||
S3Bucket: config.LibraryProductionS3Bucket,
|
||||
TenantID: config.LibraryProductionTenantID,
|
||||
DatabaseSet: config.LibraryProductionDatabaseURL != "",
|
||||
},
|
||||
}
|
||||
store, ok := config.Store.(interface {
|
||||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
LibraryRestoreVerifications(context.Context, string, int) ([]governance.LibraryRestoreVerification, error)
|
||||
})
|
||||
if !ok {
|
||||
view.Reason = "当前部署没有恢复验证存储:无法读取证据"
|
||||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||||
return
|
||||
}
|
||||
latest, found, err := store.LatestLibraryRestoreVerification(r.Context(), scope)
|
||||
if err != nil {
|
||||
view.Reason = "读取恢复验证证据失败:" + err.Error()
|
||||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||||
return
|
||||
}
|
||||
if found {
|
||||
view.Latest = &latest
|
||||
view.JobID = latest.JobID
|
||||
}
|
||||
if history, err := store.LibraryRestoreVerifications(r.Context(), scope, 10); err == nil {
|
||||
view.History = history
|
||||
}
|
||||
view.Verified, view.Reason = governance.LibraryVerificationSatisfied(latest, scope)
|
||||
if !found {
|
||||
view.Verified = false
|
||||
view.Reason = "还没有对当前目标做过恢复验证:点「跑一次恢复验证」(真实备份 → canary → 恢复 → 复核)"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, view)
|
||||
}
|
||||
|
||||
// withRestoreEvidence 把"当前目标是否有真实恢复证据"注入 Config 的副本,
|
||||
// 让同步/恢复闸口在同一个请求里按证据判断,而不是读一个可能被手填的 env。
|
||||
// 读不到证据时不改写任何东西:闸口照旧因为缺证据而拒绝。
|
||||
func (c Config) withRestoreEvidence(ctx context.Context, scope string) Config {
|
||||
if ok, err := c.restoreEvidence(ctx, scope); err == nil {
|
||||
c.restoreVerified = ok
|
||||
c.restoreEvidenceReason = ""
|
||||
} else {
|
||||
c.restoreVerified = false
|
||||
c.restoreEvidenceReason = err.Error()
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// restoreEvidence 读当前目标的最近一条证据并给出它能不能支撑"可恢复"。
|
||||
func (c Config) restoreEvidence(ctx context.Context, scope string) (bool, error) {
|
||||
store, ok := c.Store.(interface {
|
||||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
})
|
||||
if !ok {
|
||||
return false, errors.New("没有已核验的生产恢复演练证据")
|
||||
}
|
||||
latest, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("读取恢复验证证据失败:%w", err)
|
||||
}
|
||||
if !found {
|
||||
return false, errors.New("当前目标还没有恢复验证证据:先在「生产保护与恢复」里跑一次恢复验证")
|
||||
}
|
||||
ok, reason := governance.LibraryVerificationSatisfied(latest, scope)
|
||||
if !ok {
|
||||
return false, errors.New(reason)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// restoreGateReasons 是闸口对"恢复可验证"的统一判断:证据优先,
|
||||
// 历史 env 覆盖仅在证据缺失时兜底(页面不再写入它)。
|
||||
func restoreGateReasons(config Config) []string {
|
||||
if config.restoreVerified || config.ProductionBackupRestoreVerified {
|
||||
return nil
|
||||
}
|
||||
reason := config.restoreEvidenceReason
|
||||
if reason == "" {
|
||||
reason = "没有已核验的生产恢复演练证据"
|
||||
}
|
||||
return []string{reason}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 恢复验证的页面契约:
|
||||
// - 只有管理员能发起(它会停写入者并在目标上写 canary);
|
||||
// - 闸口关闭 / 维护中时直接拒绝,不投一个 worker 必然失败的作业;
|
||||
// - 证据只由真实执行写入:没有证据 ⇒ verified=false,过期/换目标 ⇒ 失效。
|
||||
//
|
||||
// 这里不模拟"真跑一次恢复"(那需要真实目标);执行体本身在 internal/libraryverify,
|
||||
// 由库内 CLI 与 worker 共用,属于隔离 fixture 之外的集成范围。
|
||||
|
||||
type verificationStore struct {
|
||||
fakeLibraryStore
|
||||
verifications []governance.LibraryRestoreVerification
|
||||
}
|
||||
|
||||
func (s *verificationStore) LatestLibraryRestoreVerification(_ context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
for _, item := range s.verifications {
|
||||
if item.Scope == scope {
|
||||
return item, true, nil
|
||||
}
|
||||
}
|
||||
return governance.LibraryRestoreVerification{}, false, nil
|
||||
}
|
||||
|
||||
func (s *verificationStore) LibraryRestoreVerifications(_ context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
|
||||
items := make([]governance.LibraryRestoreVerification, 0, len(s.verifications))
|
||||
for _, item := range s.verifications {
|
||||
if item.Scope == scope {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
if limit > 0 && len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func verifyRequest(body, role, username string) *http.Request {
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/library/verifications", strings.NewReader(body))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
ctx := context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: username, Role: role})
|
||||
return request.WithContext(ctx)
|
||||
}
|
||||
|
||||
func TestLibraryVerifyRequiresAdmin(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
recorder := httptest.NewRecorder()
|
||||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleEngineer, "eng"),
|
||||
Config{Store: store, ProductionReleaseRequestsEnabled: true})
|
||||
if recorder.Code != http.StatusForbidden {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if store.createN != 0 {
|
||||
t.Fatal("non-admin must not create a verify job")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerifyRefusesWhenWriteGateClosed(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
recorder := httptest.NewRecorder()
|
||||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
|
||||
Config{Store: store, ProductionReleaseRequestsEnabled: false})
|
||||
if recorder.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
body := decodeBody(t, recorder)
|
||||
if body["code"] != "library_verify_blocked" {
|
||||
t.Fatalf("body=%v", body)
|
||||
}
|
||||
if store.createN != 0 {
|
||||
t.Fatal("closed gate must not create a verify job")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerifyEnqueuesVerifyJobWithCanaryStages(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
store.enqueueEchoRequestedID = true
|
||||
recorder := httptest.NewRecorder()
|
||||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
|
||||
Config{Store: store, ProductionReleaseRequestsEnabled: true})
|
||||
if recorder.Code != http.StatusAccepted {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if store.lastJob.Kind != governance.LibraryJobVerify || store.lastJob.Scope != "internal" {
|
||||
t.Fatalf("job=%#v", store.lastJob)
|
||||
}
|
||||
if !strings.HasPrefix(store.lastJob.ID, "libverify-internal-") {
|
||||
t.Fatalf("job id=%q", store.lastJob.ID)
|
||||
}
|
||||
// 阶段必须包含 canary:没有"写 canary 证明校验有效"这一步,验证就只是自证。
|
||||
want := governance.VerifyStages()
|
||||
if len(want) != 5 || want[2] != "canary" || want[3] != "restore" || want[4] != "finalize" {
|
||||
t.Fatalf("verify stages=%v", want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerificationsWithoutEvidenceIsNotVerified(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
body := decodeBody(t, recorder)
|
||||
if body["verified"] != false {
|
||||
t.Fatalf("verified must be false without evidence: %v", body)
|
||||
}
|
||||
if reason, _ := body["reason"].(string); !strings.Contains(reason, "还没有对当前目标做过恢复验证") {
|
||||
t.Fatalf("reason=%q", reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerificationsWithPassedEvidenceIsVerified(t *testing.T) {
|
||||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
ID: "libverify-internal-job1", Scope: "internal", Status: governance.LibraryVerificationPassed,
|
||||
SnapshotID: "snap-1", JobID: "job1", StartedAt: time.Now().UTC().Add(-time.Minute),
|
||||
FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
body := decodeBody(t, recorder)
|
||||
if body["verified"] != true {
|
||||
t.Fatalf("verified must be true with passed evidence: %v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerificationsFailedEvidenceIsNotVerified(t *testing.T) {
|
||||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
ID: "libverify-internal-job2", Scope: "internal", Status: governance.LibraryVerificationFailed,
|
||||
SnapshotID: "snap-2", JobID: "job2", Failure: "恢复后内容仍与预备份不一致",
|
||||
StartedAt: time.Now().UTC().Add(-time.Minute), FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
body := decodeBody(t, recorder)
|
||||
if body["verified"] != false {
|
||||
t.Fatalf("failed evidence must not verify: %v", body)
|
||||
}
|
||||
if reason, _ := body["reason"].(string); !strings.Contains(reason, "恢复后内容仍与预备份不一致") {
|
||||
t.Fatalf("reason=%q", reason)
|
||||
}
|
||||
}
|
||||
|
||||
// 同步闸口必须消费同一 target 的证据:有 passed 证据即放行,
|
||||
// 只有失败证据时仍然拒绝并给出失败原因(不是笼统"没有证据")。
|
||||
func TestLibrarySyncGateConsumesRestoreEvidence(t *testing.T) {
|
||||
base := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40}
|
||||
base.Store = &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1",
|
||||
JobID: "job1", FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
allowed := librarySyncGateWithGrant(base.withRestoreEvidence(context.Background(), "internal"),
|
||||
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
|
||||
if !allowed.SyncAllowed {
|
||||
t.Fatalf("gate should allow with passed evidence: %v", allowed.Reasons)
|
||||
}
|
||||
noEvidence := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
|
||||
Store: &verificationStore{}}.withRestoreEvidence(context.Background(), "internal")
|
||||
blocked := librarySyncGateWithGrant(noEvidence,
|
||||
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
|
||||
if blocked.SyncAllowed {
|
||||
t.Fatalf("gate without env and without evidence must block: %v", blocked.Reasons)
|
||||
}
|
||||
if len(blocked.Reasons) == 0 || !strings.Contains(strings.Join(blocked.Reasons, ";"), "恢复验证") {
|
||||
t.Fatalf("blocked reasons=%v", blocked.Reasons)
|
||||
}
|
||||
failedConfig := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
|
||||
Store: &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-2",
|
||||
JobID: "job2", Failure: "canary 后校验仍通过", FinishedAt: time.Now().UTC(),
|
||||
}}}}
|
||||
reason := strings.Join(restoreGateReasons(failedConfig.withRestoreEvidence(context.Background(), "internal")), ";")
|
||||
if !strings.Contains(reason, "canary 后校验仍通过") {
|
||||
t.Fatalf("失败证据的失败原因必须出现在闸口:%q", reason)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/pgdsn"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
@@ -33,6 +34,55 @@ type productionBindingView struct {
|
||||
ProductionBaseURL string `json:"production_base_url"`
|
||||
ProductionConnected bool `json:"production_connected"`
|
||||
ProductionMessage string `json:"production_message"`
|
||||
// LegacyScope 说明这条记录属于哪条发布路径。整库发布(封存/同步/恢复)不读它,
|
||||
// 只读下面的 WholeTarget;页面必须把两者分开写,否则会把旧文档级目标当成整库目标。
|
||||
LegacyScope string `json:"legacy_scope"`
|
||||
// WholeTarget 是整库发布真正写入的目标(来自部署环境 LIBRARY_PRODUCTION_*),
|
||||
// 与上面可编辑的文档级绑定是两条独立路径。
|
||||
WholeTarget libraryPublishTargetView `json:"whole_target"`
|
||||
}
|
||||
|
||||
// libraryPublishTargetView 是"整库发布到底写哪儿"的只读投影。
|
||||
// 为什么由后端给出:页面上的 kb 名称/文档数来自生产端列表接口,而整库写入用的是
|
||||
// 另一组环境配置;让页面自己拼会把两条路径混成一个。
|
||||
type libraryPublishTargetView struct {
|
||||
Configured bool `json:"configured"`
|
||||
Source string `json:"source"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
InternalKBID string `json:"internal_kb_id"`
|
||||
ExternalKBID string `json:"external_kb_id"`
|
||||
Database string `json:"database"`
|
||||
ElasticIndex string `json:"elastic_index"`
|
||||
ObjectBucket string `json:"object_bucket"`
|
||||
MissingReason string `json:"missing_reason"`
|
||||
}
|
||||
|
||||
// libraryPublishTarget 从 config 派生整库目标;缺项照实说缺哪一项,不猜默认值。
|
||||
func (config Config) libraryPublishTarget() libraryPublishTargetView {
|
||||
view := libraryPublishTargetView{
|
||||
Source: "部署环境 LIBRARY_PRODUCTION_*",
|
||||
TenantID: config.LibraryProductionTenantID,
|
||||
InternalKBID: config.LibraryProductionInternalKB,
|
||||
ExternalKBID: config.LibraryProductionExternalKB,
|
||||
Database: pgdsn.Endpoint(config.LibraryProductionDatabaseURL),
|
||||
ElasticIndex: config.LibraryProductionIndex,
|
||||
ObjectBucket: config.LibraryProductionS3Bucket,
|
||||
}
|
||||
missing := make([]string, 0, 4)
|
||||
if config.LibraryProductionDatabaseURL == "" {
|
||||
missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL")
|
||||
}
|
||||
if view.InternalKBID == "" {
|
||||
missing = append(missing, "LIBRARY_PRODUCTION_KB_INTERNAL")
|
||||
}
|
||||
if config.LibraryProductionIndex == "" {
|
||||
missing = append(missing, "LIBRARY_PRODUCTION_INDEX")
|
||||
}
|
||||
view.Configured = len(missing) == 0
|
||||
if len(missing) > 0 {
|
||||
view.MissingReason = "未配置:" + strings.Join(missing, "、")
|
||||
}
|
||||
return view
|
||||
}
|
||||
|
||||
// productionTargetBaseURL is the production endpoint the picker reads from: the
|
||||
@@ -79,6 +129,10 @@ func (config Config) productionBindingPayload(ctx context.Context) (productionBi
|
||||
InternalKBName: names[storedInternal], ExternalKBName: names[storedExternal],
|
||||
UpdatedBy: updatedBy, ProductionBaseURL: config.productionTargetBaseURL(ctx),
|
||||
ProductionConnected: message == "", ProductionMessage: message,
|
||||
// 这张表服务的是"文档级发布"(单篇文档写生产)。整库发布读的是部署环境
|
||||
// LIBRARY_PRODUCTION_*,两者不能互相顶替,所以页面必须分开显示。
|
||||
LegacyScope: "文档级发布(单篇文档写生产)· 旧路径",
|
||||
WholeTarget: config.libraryPublishTarget(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -178,3 +178,50 @@ func TestProductionBindingOptionsReportUnavailableReason(t *testing.T) {
|
||||
t.Fatalf("status = %d body %s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 页面必须说清"整库发布真正写哪个 KB/端点"。这条投影只能来自部署配置,
|
||||
// 不能从可编辑的文档级绑定推出来,也不能把口令带进响应。
|
||||
func TestLibraryPublishTargetIsDerivedFromDeploymentConfig(t *testing.T) {
|
||||
config := Config{
|
||||
// 只放形状,不放任何可用凭据:口令段是占位符,主机是 .invalid。
|
||||
LibraryProductionDatabaseURL: "postgres://governance@db.invalid:5432/weknora_test?sslmode=disable",
|
||||
LibraryProductionInternalKB: "a8e9c8ad-411c-4c53-8421-599d29e9b179",
|
||||
LibraryProductionExternalKB: "2e980686-2233-4c4e-b5a6-8b0069477b8e",
|
||||
LibraryProductionTenantID: 10000,
|
||||
LibraryProductionIndex: "weknora_test_chunks",
|
||||
LibraryProductionS3Bucket: "dobot-weknora-test",
|
||||
}
|
||||
target := config.libraryPublishTarget()
|
||||
if !target.Configured || target.MissingReason != "" {
|
||||
t.Fatalf("完整配置应判为已配置: %+v", target)
|
||||
}
|
||||
if target.InternalKBID != "a8e9c8ad-411c-4c53-8421-599d29e9b179" || target.TenantID != 10000 {
|
||||
t.Fatalf("必须回显实际整库目标而不是占位: %+v", target)
|
||||
}
|
||||
if target.Database != "db.invalid:5432/weknora_test" {
|
||||
t.Fatalf("端点应只含 host:port/db,得到 %q", target.Database)
|
||||
}
|
||||
// 回显只能是端点:连接串里的用户信息(口令、账号)一个字符都不能进响应。
|
||||
if strings.Contains(target.Database, "@") || strings.Contains(target.Source, "@") ||
|
||||
strings.Contains(target.Source, "postgres://") {
|
||||
t.Fatalf("响应里不能出现连接串用户信息: %+v", target)
|
||||
}
|
||||
if target.ElasticIndex != "weknora_test_chunks" || target.ObjectBucket != "dobot-weknora-test" {
|
||||
t.Fatalf("索引与桶也要显示: %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryPublishTargetNamesMissingConfigInsteadOfGuessing(t *testing.T) {
|
||||
target := Config{}.libraryPublishTarget()
|
||||
if target.Configured {
|
||||
t.Fatal("缺配置时不能报告已配置")
|
||||
}
|
||||
for _, key := range []string{"LIBRARY_PRODUCTION_DATABASE_URL", "LIBRARY_PRODUCTION_KB_INTERNAL"} {
|
||||
if !strings.Contains(target.MissingReason, key) {
|
||||
t.Fatalf("缺失说明应点名 %s,得到 %q", key, target.MissingReason)
|
||||
}
|
||||
}
|
||||
if target.InternalKBID != "" || target.Database != "(未配置)" {
|
||||
t.Fatalf("缺配置就不要编造目标: %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,6 +181,9 @@ type Config struct {
|
||||
LibraryProductionInternalKB string
|
||||
LibraryProductionExternalKB string
|
||||
LibraryProductionTenantID uint64
|
||||
// 整库发布还落在这些后端上;页面要显示"到底写哪儿",只给 KB ID 不够。
|
||||
LibraryProductionIndex string
|
||||
LibraryProductionS3Bucket string
|
||||
// LibraryScopeProbe 让测试注入"目标整库是否为空"的探测;留空即用真实连接。
|
||||
// 它不是运行时开关:生产部署不设置它,那次探测仍然直连目标库逐表计数。
|
||||
LibraryScopeProbe func(ctx context.Context, scope string) (bool, map[string]int64, error)
|
||||
@@ -198,8 +201,13 @@ type Config struct {
|
||||
RequireWeKnoraReady bool
|
||||
ProductionReleaseRequestsEnabled bool
|
||||
// TestAutoReviewPolicy 只用于展示:让运维在界面上看到"现在自动批准到什么程度"。
|
||||
TestAutoReviewPolicy string
|
||||
ProductionBackupRestoreVerified bool
|
||||
TestAutoReviewPolicy string
|
||||
ProductionBackupRestoreVerified bool
|
||||
// restoreVerified / restoreEvidenceReason 由请求期的证据读取填入(withRestoreEvidence):
|
||||
// 闸口按"当前目标是否有真实恢复验证证据"判断,而不是按可手填的 env。
|
||||
// 它们不可由配置注入,避免把证据变成另一个开关。
|
||||
restoreVerified bool
|
||||
restoreEvidenceReason string
|
||||
ProductionReleaseAuthorizerRef string
|
||||
ProductionReleaseAuthorizationToken string
|
||||
// DingtalkSources 枚举「我账号下能看到」的钉钉知识库与钉盘空间(只读,dws)。
|
||||
@@ -556,6 +564,14 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
mux.HandleFunc("POST /api/v1/library/baselines", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryBaselineEnqueue(w, r, config)
|
||||
})
|
||||
// 恢复验证:真实备份 → canary → 恢复 → 复核,通过后证据按当前目标登记,
|
||||
// 同步闸口据此放行(不再需要页面/环境变量手填 verified)。
|
||||
mux.HandleFunc("POST /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryVerifyEnqueue(w, r, config)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryVerifications(w, r, config)
|
||||
})
|
||||
// 全部 main 提交时间线(读本库缓存,不直接打 Gitea)
|
||||
mux.HandleFunc("GET /api/v1/version-history", func(w http.ResponseWriter, r *http.Request) {
|
||||
versionHistoryList(w, r, config)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package main
|
||||
package libraryverify
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -0,0 +1,266 @@
|
||||
// Package libraryverify 在目标实例上做一次**真实**的恢复验证(原 cmd/librarydrill 的实现):
|
||||
//
|
||||
// 维护隔离(关入口/停写入者/证明排空)
|
||||
// → 捕获目标整库(真预备份,落归档)
|
||||
// → 校验目标 == 预备份(基线一致)
|
||||
// → 写入一个 canary(证明"校验确实能发现内容变化")
|
||||
// → 按预备份恢复
|
||||
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
|
||||
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
|
||||
//
|
||||
// 为什么搬进 internal:页面要能点一次"恢复验证",worker 必须在进程内执行同一份实现,
|
||||
// 不能再要求运维 SSH 上去跑 CLI。命令行使同一个 Run,不允许两份实现。
|
||||
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境,不进命令行、不打印。
|
||||
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
|
||||
package libraryverify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/libraryfreeze"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarystack"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
type Verification struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Verified bool `json:"verified"`
|
||||
Error string `json:"error,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Status string `json:"status"` // passed | failed | maintenance_held
|
||||
Scope string `json:"scope"`
|
||||
Target string `json:"target"`
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Project string `json:"project"`
|
||||
Counts map[string]int64 `json:"counts"`
|
||||
BaselineCheck *Verification `json:"baseline_check"`
|
||||
DriftCheck *Verification `json:"drift_check"` // 必须失败:证明确实能发现差异
|
||||
RestoreCheck *Verification `json:"restore_check"`
|
||||
Canary string `json:"canary"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
Stage string `json:"stage,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
Until map[string]string `json:"until,omitempty"`
|
||||
}
|
||||
|
||||
// Progress 把演练的阶段推进交给调用方(worker 写 library_job_stages,CLI 打日志)。
|
||||
type Progress func(stage, state, detail string)
|
||||
|
||||
func (p Progress) report(stage, state, detail string) {
|
||||
if p != nil {
|
||||
p(stage, state, detail)
|
||||
}
|
||||
}
|
||||
|
||||
// Run 执行一次演练。jobID 同时是维护窗口的持有者标识,必须与业务作业 id 一致,
|
||||
// 否则崩溃恢复时窗口与作业对不上。
|
||||
func Run(ctx context.Context, config appconfig.Config, scope, jobID string, progress Progress) (*Report, error) {
|
||||
if scope != "internal" && scope != "external" {
|
||||
return nil, errors.New("scope must be internal or external")
|
||||
}
|
||||
report := &Report{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
|
||||
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
|
||||
fail := func(stage string, cause error, held bool) (*Report, error) {
|
||||
report.Stage = stage
|
||||
report.Failure = cause.Error()
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
if held {
|
||||
report.Status = "maintenance_held"
|
||||
}
|
||||
progress.report(stage, "failed", cause.Error())
|
||||
return report, cause
|
||||
}
|
||||
|
||||
progress.report("capture", "running", "打开目标栈与维护管理器")
|
||||
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
|
||||
if err != nil {
|
||||
return fail("capture", err, false)
|
||||
}
|
||||
defer target.Close()
|
||||
manager, err := librarystack.OpenMaintenance(config)
|
||||
if err != nil {
|
||||
return fail("capture", err, false)
|
||||
}
|
||||
defer manager.Close()
|
||||
if err := manager.VerifyTopology(ctx); err != nil {
|
||||
return fail("capture", err, false)
|
||||
}
|
||||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||||
if err != nil {
|
||||
return fail("capture", err, false)
|
||||
}
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
|
||||
var (
|
||||
snapshotID string
|
||||
counts map[string]int64
|
||||
canary string
|
||||
contentTouched bool
|
||||
baseline Verification
|
||||
drift Verification
|
||||
restored Verification
|
||||
)
|
||||
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
||||
start := time.Now()
|
||||
current, err := target.Current(ctx)
|
||||
if err != nil {
|
||||
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
|
||||
}
|
||||
counts = current.Counts
|
||||
report.Counts = counts
|
||||
// 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。
|
||||
backup, err := backups.Prepare(ctx, "", current)
|
||||
if err != nil {
|
||||
return "", errors.New("捕获目标整库失败:" + err.Error())
|
||||
}
|
||||
snapshotID = backup.SnapshotID
|
||||
report.SnapshotID = snapshotID
|
||||
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
|
||||
return "", errors.New("记录预备份身份失败:" + err.Error())
|
||||
}
|
||||
progress.report("capture", "succeeded",
|
||||
fmt.Sprintf("已捕获目标整库快照 %s(%d 字节,用时 %s)", snapshotID, backup.Bytes, time.Since(start).Round(time.Second)))
|
||||
log.Printf("verify: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
|
||||
|
||||
// 2) 基线校验:预备份必须与目标完全一致。
|
||||
progress.report("verify", "running", "逐表/索引/资源比对目标与预备份")
|
||||
baseline = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||||
baseline.Error = err.Error()
|
||||
report.BaselineCheck = &baseline
|
||||
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
|
||||
}
|
||||
baseline.Verified = true
|
||||
report.BaselineCheck = &baseline
|
||||
progress.report("verify", "succeeded", "预备份与目标逐类一致")
|
||||
|
||||
// Canary 是隔离的标签行,空目标上也一样:绝不为了"让演练能跑"去种文档或改既有行。
|
||||
var random [16]byte
|
||||
if _, err := rand.Read(random[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
canary = fmt.Sprintf("%x", random[:])
|
||||
identityValue := target.Identity()
|
||||
var exists bool
|
||||
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if exists {
|
||||
return "", errors.New("canary identity collision; no write attempted")
|
||||
}
|
||||
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
|
||||
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
|
||||
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
|
||||
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
|
||||
report.Canary = canary
|
||||
progress.report("canary", "running", "写入一个隔离 canary,验证校验能发现差异")
|
||||
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
|
||||
func(ctx context.Context) error {
|
||||
if err := window.MarkContentRisk(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
contentTouched = true
|
||||
return nil
|
||||
}, func(ctx context.Context) error {
|
||||
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
|
||||
return err
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("canary preparation/write failed: %w", err)
|
||||
}
|
||||
drift = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
||||
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
|
||||
report.DriftCheck = &drift
|
||||
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
|
||||
} else {
|
||||
drift.Error = verifyErr.Error()
|
||||
}
|
||||
report.DriftCheck = &drift
|
||||
progress.report("canary", "succeeded", "校验按预期发现了 canary 差异")
|
||||
log.Printf("verify: drift detected as expected (%s)", drift.Error)
|
||||
|
||||
// 4) 按预备份恢复。恢复只删掉持久写集合里那个新 canary 身份。
|
||||
progress.report("restore", "running", "按预备份恢复目标整库")
|
||||
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
|
||||
WriteSet: &ledger}); err != nil {
|
||||
return "", errors.New("按预备份恢复失败:" + err.Error())
|
||||
}
|
||||
// 5) 恢复后逐类复核:必须与预备份完全一致。
|
||||
restored = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||||
restored.Error = err.Error()
|
||||
report.RestoreCheck = &restored
|
||||
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
|
||||
}
|
||||
restored.Verified = true
|
||||
report.RestoreCheck = &restored
|
||||
progress.report("restore", "succeeded", "恢复后与预备份逐类一致")
|
||||
log.Printf("verify: restored and verified snapshot=%s", snapshotID)
|
||||
return snapshotID, nil
|
||||
}
|
||||
|
||||
progress.report("finalize", "running", "关闭维护窗口并按原状态恢复目标")
|
||||
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
||||
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
||||
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
|
||||
func(error) bool { return !contentTouched || restored.Verified })
|
||||
|
||||
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
||||
if windowErr != nil || window.State != "succeeded" {
|
||||
cause := windowErr
|
||||
if cause == nil {
|
||||
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
|
||||
}
|
||||
if window.State == "held" {
|
||||
return fail("finalize", cause, true)
|
||||
}
|
||||
return fail("finalize", cause, false)
|
||||
}
|
||||
report.Status = "passed"
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
progress.report("finalize", "succeeded", "维护窗口已关闭,目标恢复原运行状态")
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error {
|
||||
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
|
||||
if err := ledger.Seal(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := mark(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return write(ctx)
|
||||
}
|
||||
|
||||
// RecoverWindows 把非终态的维护窗口按原运行状态恢复(服务起来、入口验证)。
|
||||
// 演练失败但还没写目标内容时,用它把目标从"停住"状态救回来;治理记录由作业/接口那侧负责。
|
||||
func RecoverWindows(config appconfig.Config) error {
|
||||
manager, err := librarystack.OpenMaintenance(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer manager.Close()
|
||||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
return coordinator.Recover(context.Background())
|
||||
}
|
||||
@@ -33,7 +33,8 @@ func (s *Store) createLibraryDispatch(ctx context.Context, job governance.Librar
|
||||
return created, true, nil
|
||||
}
|
||||
kind := "library." + string(created.Kind)
|
||||
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore && created.Kind != governance.LibraryJobBaseline {
|
||||
if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore &&
|
||||
created.Kind != governance.LibraryJobBaseline && created.Kind != governance.LibraryJobVerify {
|
||||
return created, reused, errors.New("unsupported library dispatch kind")
|
||||
}
|
||||
queueID, kind, raw, err := governance.LibraryQueueRequest(handoff, created.ID, created.Scope, created.TargetSnapshotID, kind)
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 恢复验证证据:只由 worker 在真实演练结束后写入;这里只做落库与按当前目标读取。
|
||||
//
|
||||
// 读取永远带 target_id = 当前部署目标:换库/换索引/换桶之后旧证据自动失效,
|
||||
// 不需要任何人去"记得把开关关掉"。
|
||||
|
||||
// SaveLibraryRestoreVerification 幂等登记一次演练结果(同一 target+scope+job 只留一条)。
|
||||
func (s *Store) SaveLibraryRestoreVerification(ctx context.Context, verification governance.LibraryRestoreVerification) error {
|
||||
if verification.ID == "" || verification.JobID == "" {
|
||||
return fmt.Errorf("restore verification requires id and job id")
|
||||
}
|
||||
if verification.Scope != "internal" && verification.Scope != "external" {
|
||||
return fmt.Errorf("restore verification scope %q is invalid", verification.Scope)
|
||||
}
|
||||
if verification.Status != governance.LibraryVerificationPassed && verification.Status != governance.LibraryVerificationFailed {
|
||||
return fmt.Errorf("restore verification status %q is invalid", verification.Status)
|
||||
}
|
||||
report, err := json.Marshal(verification.Report)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode restore verification report: %w", err)
|
||||
}
|
||||
_, err = s.db.Exec(ctx, `
|
||||
INSERT INTO library_restore_verifications (id, target_id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (target_id, scope, job_id) DO UPDATE
|
||||
SET status = EXCLUDED.status, snapshot_id = EXCLUDED.snapshot_id, failure = EXCLUDED.failure,
|
||||
report = EXCLUDED.report, started_at = EXCLUDED.started_at, finished_at = EXCLUDED.finished_at`,
|
||||
verification.ID, s.libraryTargetID, verification.Scope, verification.Status,
|
||||
verification.SnapshotID, verification.JobID, verification.Failure, report,
|
||||
verification.StartedAt.UTC(), verification.FinishedAt.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("save restore verification: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LatestLibraryRestoreVerification 按完成时间取当前目标该 scope 的最近一条证据。
|
||||
// 没有记录返回 (zero,false,nil):那不是错误,页面据此说"还没有验证过"。
|
||||
func (s *Store) LatestLibraryRestoreVerification(ctx context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
var (
|
||||
item governance.LibraryRestoreVerification
|
||||
report []byte
|
||||
)
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT id, scope, status, snapshot_id, job_id, failure, report, started_at, finished_at
|
||||
FROM library_restore_verifications
|
||||
WHERE target_id=$1 AND scope=$2
|
||||
ORDER BY finished_at DESC, created_at DESC
|
||||
LIMIT 1`, s.libraryTargetID, scope).Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID,
|
||||
&item.JobID, &item.Failure, &report, &item.StartedAt, &item.FinishedAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return governance.LibraryRestoreVerification{}, false, nil
|
||||
}
|
||||
return governance.LibraryRestoreVerification{}, false, fmt.Errorf("load restore verification: %w", err)
|
||||
}
|
||||
item.Report = decodeEvidenceObject(report)
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
// LibraryRestoreVerifications 是页面用的轻量清单(最近 N 条),按目标过滤。
|
||||
func (s *Store) LibraryRestoreVerifications(ctx context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, scope, status, snapshot_id, job_id, failure, started_at, finished_at
|
||||
FROM library_restore_verifications
|
||||
WHERE target_id=$1 AND scope=$2
|
||||
ORDER BY finished_at DESC, created_at DESC
|
||||
LIMIT $3`, s.libraryTargetID, scope, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list restore verifications: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]governance.LibraryRestoreVerification, 0, limit)
|
||||
for rows.Next() {
|
||||
var item governance.LibraryRestoreVerification
|
||||
if err := rows.Scan(&item.ID, &item.Scope, &item.Status, &item.SnapshotID, &item.JobID,
|
||||
&item.Failure, &item.StartedAt, &item.FinishedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
DROP INDEX IF EXISTS library_restore_verifications_latest;
|
||||
DROP TABLE IF EXISTS library_restore_verifications;
|
||||
|
||||
-- 先把 verify 作业清掉(它没有历史数据,只可能来自本版本),再恢复旧的 kind 约束。
|
||||
DELETE FROM library_jobs WHERE kind = 'verify';
|
||||
ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check;
|
||||
ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check
|
||||
CHECK (kind IN ('seal','sync','restore','baseline'));
|
||||
|
||||
DROP INDEX IF EXISTS library_jobs_write_target;
|
||||
CREATE UNIQUE INDEX library_jobs_write_target
|
||||
ON library_jobs(scope)
|
||||
WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline');
|
||||
@@ -0,0 +1,37 @@
|
||||
-- 恢复验证成为一种独立业务作业,并且它的证据只能由真实演练写入。
|
||||
--
|
||||
-- 为什么需要:闸门原来读 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个纯 env 开关,
|
||||
-- 谁把它写成 true 都能"证明"生产可恢复;页面也没有入口去真跑一次恢复。
|
||||
-- 现在 verified 只能来自 library_restore_verifications 里 passed 的真实记录。
|
||||
--
|
||||
-- verify 与 sync/restore/baseline 共用同一把写互斥:它同样要停写入者、开维护窗口、
|
||||
-- 在目标上写 canary 再恢复,不能和别的写作业并发。
|
||||
ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check;
|
||||
ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check
|
||||
CHECK (kind IN ('seal','sync','restore','baseline','verify'));
|
||||
|
||||
DROP INDEX IF EXISTS library_jobs_write_target;
|
||||
CREATE UNIQUE INDEX library_jobs_write_target
|
||||
ON library_jobs(scope)
|
||||
WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline','verify');
|
||||
|
||||
-- target_id 是 appconfig.LibraryTargetIdentity():换目标(库/索引/桶/实例)后旧证据不再适用,
|
||||
-- 这是"证据必须对得上当前目标"的落库形态。
|
||||
CREATE TABLE IF NOT EXISTS library_restore_verifications (
|
||||
id text PRIMARY KEY,
|
||||
target_id text NOT NULL CHECK (target_id <> ''),
|
||||
scope text NOT NULL CHECK (scope IN ('internal','external')),
|
||||
status text NOT NULL CHECK (status IN ('passed','failed')),
|
||||
snapshot_id text NOT NULL DEFAULT '',
|
||||
job_id text NOT NULL CHECK (job_id <> ''),
|
||||
failure text NOT NULL DEFAULT '',
|
||||
report jsonb,
|
||||
started_at timestamptz NOT NULL,
|
||||
finished_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
-- 同一作业只留一条;重复上报是幂等重放,不是新证据。
|
||||
CONSTRAINT library_restore_verifications_job UNIQUE (target_id, scope, job_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS library_restore_verifications_latest
|
||||
ON library_restore_verifications (target_id, scope, status, finished_at DESC);
|
||||
@@ -11,8 +11,15 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 78 {
|
||||
t.Fatalf("migration count = %d, want 78", len(migrations))
|
||||
if len(migrations) != 79 {
|
||||
t.Fatalf("migration count = %d, want 79", len(migrations))
|
||||
}
|
||||
// 79 是恢复验证:verify 成为独立业务作业并与写作业共用互斥,
|
||||
// 证据按 target_id 落库(换目标旧证据自动失效)。
|
||||
if verification := migrations[78]; verification.Version != 79 ||
|
||||
!strings.Contains(verification.UpSQL, "library_restore_verifications") ||
|
||||
!strings.Contains(verification.UpSQL, "'verify'") {
|
||||
t.Fatalf("migration 79 = %#v", verification)
|
||||
}
|
||||
// 78 是 DS01 恢复标记派发台账:唯一 key、单调状态与激活登记;只动治理库。
|
||||
if dispatch := migrations[77]; dispatch.Version != 78 ||
|
||||
|
||||
@@ -22,6 +22,7 @@ const base: BackupSchedule = {
|
||||
last_applied_summary: '每天 03:30 · 保留 7 天 · 每月归档 3 份',
|
||||
host_configured: true,
|
||||
host_target: 'weknora-test',
|
||||
host_endpoint: '100.85.111.121:22',
|
||||
host_reachable: true,
|
||||
host_error: '',
|
||||
host_schedule_enabled: true,
|
||||
@@ -88,6 +89,9 @@ describe('BackupScheduleView', () => {
|
||||
'重新下发当前设置',
|
||||
'立即备份一次',
|
||||
'weknora-test · 已连接',
|
||||
'拨号地址',
|
||||
'100.85.111.121:22',
|
||||
'全库,不是 scope 快照',
|
||||
'每天 03:30 · 保留 7 天 · 每月 3 份',
|
||||
'已一致',
|
||||
'20261009T052909Z',
|
||||
@@ -120,6 +124,28 @@ describe('BackupScheduleView', () => {
|
||||
expect(html).toContain('disabled')
|
||||
})
|
||||
|
||||
it('连不上主机:报连接失败与拨号地址,脚本状态写未知而不是"缺失"', () => {
|
||||
const html = view({
|
||||
host_reachable: false,
|
||||
host_endpoint: '120.76.217.51:22',
|
||||
host_error:
|
||||
'目标主机 weknora-test 不可达(ssh: connect to host 120.76.217.51 port 22: Operation timed out);未执行任何主机侧脚本,脚本与计划状态未知(拨号地址 120.76.217.51:22)',
|
||||
in_sync: false,
|
||||
host_script_present: false,
|
||||
})
|
||||
expect(html).toContain('weknora-test · 连接失败')
|
||||
expect(html).toContain('120.76.217.51:22')
|
||||
expect(html).toContain('未知(未连上主机,未执行主机侧脚本)')
|
||||
expect(html).not.toContain('缺失')
|
||||
})
|
||||
|
||||
it('已连上且脚本真的不在:这时才显示"缺失"', () => {
|
||||
const html = view({ host_reachable: true, host_script_present: false, in_sync: false })
|
||||
expect(html).toContain('weknora-test · 已连接')
|
||||
expect(html).toContain('缺失')
|
||||
expect(html).not.toContain('未知(未连上主机')
|
||||
})
|
||||
|
||||
it('待下发(主机上的计划与设置不一致)时提示', () => {
|
||||
const html = view({ host_schedule_time: '05:00', in_sync: false })
|
||||
expect(html).toContain('待下发')
|
||||
|
||||
@@ -91,6 +91,12 @@ export function BackupScheduleView(props: BackupScheduleViewProps) {
|
||||
: schedule.host_reachable
|
||||
? `${schedule.host_target} · 已连接`
|
||||
: `${schedule.host_target} · 连接失败`
|
||||
// 只有真的连上并读回主机状态时,才能说脚本在不在;连不上是"未知"。
|
||||
const scriptLine = !schedule.host_reachable
|
||||
? '未知(未连上主机,未执行主机侧脚本)'
|
||||
: schedule.host_script_present
|
||||
? '存在'
|
||||
: '缺失'
|
||||
|
||||
return (
|
||||
<section className="settings-panel wide storage-settings-panel" id="settings-storage">
|
||||
@@ -115,11 +121,12 @@ export function BackupScheduleView(props: BackupScheduleViewProps) {
|
||||
<dl>
|
||||
<dt>目标主机</dt>
|
||||
<dd>{hostLine}</dd>
|
||||
<dt>拨号地址</dt>
|
||||
<dd>{schedule.host_endpoint || '—'}</dd>
|
||||
<dt>备份脚本</dt>
|
||||
<dd>{scriptLine}</dd>
|
||||
<dt>主机 cron</dt>
|
||||
<dd>
|
||||
{schedule.host_cron_service ? `服务 ${schedule.host_cron_service}` : '—'}
|
||||
{schedule.host_script_present ? '' : ' · 备份脚本缺失'}
|
||||
</dd>
|
||||
<dd>{schedule.host_cron_service ? `服务 ${schedule.host_cron_service}` : '—'}</dd>
|
||||
<dt>最近一次备份</dt>
|
||||
<dd>
|
||||
{schedule.last_run_stamp || '—'}
|
||||
@@ -213,8 +220,12 @@ export function BackupScheduleView(props: BackupScheduleViewProps) {
|
||||
{busy === 'apply' ? '正在下发…' : '重新下发当前设置'}
|
||||
</button>
|
||||
<button onClick={onRunNow} disabled={busy !== null || !schedule.host_configured || schedule.running}>
|
||||
{schedule.running ? '备份进行中…' : '立即备份一次'}
|
||||
{schedule.running ? '备份进行中…' : '立即备份一次(全库,不是 scope 快照)'}
|
||||
</button>
|
||||
<p className="hint">
|
||||
「立即备份一次」执行的是主机上的整库逻辑备份(该主机上全部业务库的 pg_dump 目录),
|
||||
<strong>不是</strong>本次对内/对外 scope 的整库快照;scope 快照在「发布管理 → 整库版本」里封存。
|
||||
</p>
|
||||
{message ? (
|
||||
<p className="success-message" role="status">
|
||||
{message}
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { renderToStaticMarkup } from 'react-dom/server'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { ProductionBindingsView } from './ProductionBindingsSection'
|
||||
import type { ProductionBindings } from './productionBindings'
|
||||
|
||||
const bindings = (overrides: Partial<ProductionBindings> = {}): ProductionBindings => ({
|
||||
internal_kb_id: '0b217e8a-2a65-40a3-ab8f-34ac5c7a6f05',
|
||||
external_kb_id: '9a1e5c11-0000-4000-8000-000000000002',
|
||||
internal_kb_name: '对内知识库-wiki',
|
||||
external_kb_name: '对外知识库-wiki',
|
||||
updated_by: 'yaozhanghao',
|
||||
production_base_url: 'https://weknora-test.dobot.cn',
|
||||
production_connected: true,
|
||||
production_message: '',
|
||||
legacy_scope: '文档级发布(单篇文档写生产)· 旧路径',
|
||||
whole_target: {
|
||||
configured: true,
|
||||
source: '部署环境 LIBRARY_PRODUCTION_*',
|
||||
tenant_id: 10000,
|
||||
internal_kb_id: 'a8e9c8ad-411c-4c53-8421-599d29e9b179',
|
||||
external_kb_id: '2e980686-2233-4c4e-b5a6-8b0069477b8e',
|
||||
database: 'pgm-wz9ohr9gk919pr1q.pg.rds.aliyuncs.com:5432/weknora_test',
|
||||
elastic_index: 'weknora_test_chunks',
|
||||
object_bucket: 'dobot-weknora-test',
|
||||
missing_reason: '',
|
||||
},
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const view = (value: ProductionBindings | null, admin = true) =>
|
||||
renderToStaticMarkup(
|
||||
<ProductionBindingsView
|
||||
bindings={value}
|
||||
options={[]}
|
||||
internalDraft={value?.internal_kb_id ?? ''}
|
||||
externalDraft={value?.external_kb_id ?? ''}
|
||||
isAdmin={admin}
|
||||
loading={false}
|
||||
saving={false}
|
||||
error={null}
|
||||
message=""
|
||||
dirty={false}
|
||||
sameSelection={false}
|
||||
onInternalChange={vi.fn()}
|
||||
onExternalChange={vi.fn()}
|
||||
onSave={vi.fn()}
|
||||
onReload={vi.fn()}
|
||||
/>
|
||||
)
|
||||
|
||||
describe('ProductionBindingsView', () => {
|
||||
it('把实际整库目标(完整 ID + 端点)显示出来,而不是只显示可编辑的文档级绑定', () => {
|
||||
const html = view(bindings())
|
||||
expect(html).toContain('整库发布目标(实际写入,只读)')
|
||||
// 完整 ID 必须可读:截断会让操作者核对不了 a8e9c8ad 与 0b217e8a 的区别。
|
||||
expect(html).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179')
|
||||
expect(html).toContain('2e980686-2233-4c4e-b5a6-8b0069477b8e')
|
||||
expect(html).toContain('pgm-wz9ohr9gk919pr1q.pg.rds.aliyuncs.com:5432/weknora_test')
|
||||
expect(html).toContain('weknora_test_chunks')
|
||||
expect(html).toContain('dobot-weknora-test')
|
||||
expect(html).toContain('部署环境 LIBRARY_PRODUCTION_*')
|
||||
})
|
||||
|
||||
it('旧文档级绑定标成旧路径,不再声称"发布写入此库"', () => {
|
||||
const html = view(bindings())
|
||||
expect(html).toContain('文档级发布(单篇文档写生产)· 旧路径')
|
||||
expect(html).toContain('仅文档级发布读取这组绑定;整库发布(封存/同步/恢复)不读它。')
|
||||
expect(html).toContain('对内文档级目标')
|
||||
expect(html).not.toContain('发布写入此库')
|
||||
})
|
||||
|
||||
it('整库目标缺配置:点名缺哪个环境变量,且不拿文档级绑定顶上去', () => {
|
||||
const html = view(
|
||||
bindings({
|
||||
whole_target: {
|
||||
configured: false,
|
||||
source: '部署环境 LIBRARY_PRODUCTION_*',
|
||||
tenant_id: 0,
|
||||
internal_kb_id: '',
|
||||
external_kb_id: '',
|
||||
database: '(未配置)',
|
||||
elastic_index: '',
|
||||
object_bucket: '',
|
||||
missing_reason: '未配置:LIBRARY_PRODUCTION_DATABASE_URL、LIBRARY_PRODUCTION_KB_INTERNAL',
|
||||
},
|
||||
})
|
||||
)
|
||||
expect(html).toContain('LIBRARY_PRODUCTION_DATABASE_URL')
|
||||
expect(html).toContain('整库目标配置不全')
|
||||
// 文档级绑定仍然显示,但必须仍是"文档级"身份,不能冒充整库目标。
|
||||
expect(html).toContain('对内文档级目标')
|
||||
})
|
||||
|
||||
it('非管理员看得到目标,但没有保存入口', () => {
|
||||
const html = view(bindings(), false)
|
||||
expect(html).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179')
|
||||
expect(html).not.toContain('保存文档级目标')
|
||||
expect(html).toContain('仅管理员可修改文档级发布目标')
|
||||
})
|
||||
})
|
||||
@@ -5,8 +5,182 @@ import { fetchProductionBindingOptions, fetchProductionBindings, saveProductionB
|
||||
import type { ProductionBindings } from './productionBindings'
|
||||
import type { KnowledgeBaseOption } from './routingBindings'
|
||||
|
||||
// 「生产知识库绑定」:发布链路的 源库 → 目标库 映射。这里选的是发布要写入的
|
||||
// 生产(目标)知识库,与上面的「测试知识库绑定」(测试端收件库)是两件事。
|
||||
// 「生产知识库绑定」:这里同时涉及两条独立路径,页面必须分开写清楚:
|
||||
// 1) 整库发布目标(封存/同步/恢复真正写入的库与端点)——来自部署配置,只读;
|
||||
// 2) 文档级绑定(旧流程:单篇文档发布写哪个库)——可在这里编辑。
|
||||
// 把 2 写成"发布写入此库"会让操作者以为整库发布会写进 0b217e8a 这类 2000+ 文档库。
|
||||
|
||||
export interface ProductionBindingsViewProps {
|
||||
bindings: ProductionBindings | null
|
||||
options: KnowledgeBaseOption[]
|
||||
internalDraft: string
|
||||
externalDraft: string
|
||||
isAdmin: boolean
|
||||
loading: boolean
|
||||
saving: boolean
|
||||
error: string | null
|
||||
message: string
|
||||
dirty: boolean
|
||||
sameSelection: boolean
|
||||
onInternalChange: (next: string) => void
|
||||
onExternalChange: (next: string) => void
|
||||
onSave: () => void
|
||||
onReload: () => void
|
||||
}
|
||||
|
||||
export function ProductionBindingsView(props: ProductionBindingsViewProps) {
|
||||
const {
|
||||
bindings,
|
||||
options,
|
||||
internalDraft,
|
||||
externalDraft,
|
||||
isAdmin,
|
||||
loading,
|
||||
saving,
|
||||
error,
|
||||
message,
|
||||
dirty,
|
||||
sameSelection,
|
||||
onInternalChange,
|
||||
onExternalChange,
|
||||
onSave,
|
||||
onReload,
|
||||
} = props
|
||||
const optionLabel = (base: KnowledgeBaseOption) => `${base.name}(${base.knowledge_count} 文档)`
|
||||
const renderSide = (label: string, boundID: string, boundName: string) => (
|
||||
<div className={`kb-binding-side ${boundID ? 'bound' : 'warn'}`}>
|
||||
<b>{label}</b>
|
||||
<span>
|
||||
{boundID ? boundName || `${boundID.slice(0, 8)}…` : '未绑定 · 按同 ID 语义发布(目标端须存在同 ID 库)'}
|
||||
</span>
|
||||
<small>{boundID ? '文档级绑定 · 只影响单篇文档发布,不改整库目标' : '未绑定:沿用测试库 ID'}</small>
|
||||
</div>
|
||||
)
|
||||
const whole = bindings?.whole_target
|
||||
const wholeRows: [string, string][] = whole
|
||||
? [
|
||||
['对内目标知识库', whole.internal_kb_id || '未配置'],
|
||||
['对外目标知识库', whole.external_kb_id || '未配置'],
|
||||
['目标数据库', whole.database || '—'],
|
||||
['检索索引', whole.elastic_index || '—'],
|
||||
['对象存储桶', whole.object_bucket || '—'],
|
||||
['租户', whole.tenant_id ? String(whole.tenant_id) : '—'],
|
||||
['配置来源', whole.source || '—'],
|
||||
]
|
||||
: []
|
||||
const editorFor = (value: string, onChange: (next: string) => void, placeholder: string) =>
|
||||
options.length > 0 ? (
|
||||
<select value={value} onChange={(event) => onChange(event.target.value)} aria-label={placeholder}>
|
||||
<option value="">{placeholder}</option>
|
||||
{options.map((base) => (
|
||||
<option key={base.id} value={base.id}>
|
||||
{optionLabel(base)}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
) : (
|
||||
<input
|
||||
value={value}
|
||||
onChange={(event) => onChange(event.target.value.trim())}
|
||||
placeholder={`${placeholder}(输入生产知识库 ID)`}
|
||||
aria-label={placeholder}
|
||||
/>
|
||||
)
|
||||
|
||||
return (
|
||||
<section className="settings-panel wide" id="settings-production-bindings">
|
||||
<div className="settings-section-heading">
|
||||
<div>
|
||||
<h2>生产知识库绑定</h2>
|
||||
<p>
|
||||
这里有两件不同的事,页面分开显示:<strong>整库发布目标</strong>(封存/同步/恢复真正写入的目标,
|
||||
由部署配置决定,只读)与<strong>文档级绑定</strong>(旧流程:单篇文档发布写哪个库,可在这里改)。
|
||||
</p>
|
||||
</div>
|
||||
<button onClick={onReload} disabled={loading}>
|
||||
{loading ? '读取中…' : '重新读取'}
|
||||
</button>
|
||||
</div>
|
||||
{error ? (
|
||||
<p className="error-message" role="alert">
|
||||
{error}
|
||||
</p>
|
||||
) : null}
|
||||
{bindings ? (
|
||||
<div className="stack" id="settings-whole-library-target">
|
||||
<h3>整库发布目标(实际写入,只读)</h3>
|
||||
{whole?.configured === false ? (
|
||||
<p className="hint">
|
||||
整库目标配置不全:{whole.missing_reason}。整库封存/同步/恢复在补齐部署配置前会被拒绝,
|
||||
页面不会用下面的文档级绑定顶上。
|
||||
</p>
|
||||
) : null}
|
||||
{wholeRows.length ? (
|
||||
<dl className="storage-settings-grid">
|
||||
{wholeRows.map(([label, value]) => (
|
||||
<div key={label}>
|
||||
<dt>{label}</dt>
|
||||
<dd>
|
||||
<code>{value}</code>
|
||||
</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
) : null}
|
||||
<p className="hint">
|
||||
这份目标来自部署环境,不在页面上编辑;改它需要改部署配置并重启 api/worker,不会由本页保存。
|
||||
</p>
|
||||
</div>
|
||||
) : null}
|
||||
{bindings ? (
|
||||
<div className="stack">
|
||||
<h3>{bindings.legacy_scope || '文档级绑定(旧路径)'}</h3>
|
||||
<p className="hint">仅文档级发布读取这组绑定;整库发布(封存/同步/恢复)不读它。</p>
|
||||
<div className="kb-binding-grid">
|
||||
{renderSide('对内文档级目标', bindings.internal_kb_id, bindings.internal_kb_name)}
|
||||
{renderSide('对外文档级目标', bindings.external_kb_id, bindings.external_kb_name)}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
{bindings && !bindings.production_connected ? (
|
||||
<p className="hint">
|
||||
生产端核对不可用:{bindings.production_message}。
|
||||
{bindings.production_base_url ? `(生产地址:${bindings.production_base_url})` : ''}
|
||||
</p>
|
||||
) : null}
|
||||
{bindings && bindings.production_connected && bindings.production_base_url ? (
|
||||
<p className="hint">
|
||||
生产端:{bindings.production_base_url}
|
||||
{bindings.updated_by ? ` · 最近修改:${bindings.updated_by}` : ''}
|
||||
</p>
|
||||
) : null}
|
||||
{isAdmin && bindings ? (
|
||||
<div className="kb-binding-editor">
|
||||
<label>
|
||||
对内文档级目标 {editorFor(internalDraft, onInternalChange, '选择对内文档级目标库')}
|
||||
</label>
|
||||
<label>
|
||||
对外文档级目标 {editorFor(externalDraft, onExternalChange, '选择对外文档级目标库')}
|
||||
</label>
|
||||
<p className="hint">保存时只改文档级绑定;整库发布目标由部署配置决定,本页不改它。</p>
|
||||
<p className="hint">保存时会向生产端核对所选的库是否真实存在;对不上会拒收,生产端不可达则只提示。</p>
|
||||
{sameSelection ? <p className="error-message">对内与对外必须绑定不同的生产知识库</p> : null}
|
||||
<button onClick={onSave} disabled={saving || !dirty || sameSelection}>
|
||||
{saving ? '正在核对并保存…' : '保存文档级目标'}
|
||||
</button>
|
||||
{message ? (
|
||||
<p className="success-message" role="status">
|
||||
{message}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
{!isAdmin ? <p className="hint">仅管理员可修改文档级发布目标。</p> : null}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
// 容器:读取 + 保存。整库目标只读展示,保存动作只写文档级绑定。
|
||||
export function ProductionBindingsSection() {
|
||||
const auth = useOptionalAuth()
|
||||
const isAdmin = auth?.identity?.role === 'admin'
|
||||
@@ -56,8 +230,8 @@ export function ProductionBindingsSection() {
|
||||
const where = `对内 ${next.internal_kb_name || next.internal_kb_id.slice(0, 8) || '未绑定'} · 对外 ${next.external_kb_name || next.external_kb_id.slice(0, 8) || '未绑定'}`
|
||||
setMessage(
|
||||
next.production_connected
|
||||
? `发布目标已更新:${where}。之后新发布的“对内/对外”内容会写入这两个生产库(已发布过的条目仍回到原目标)。`
|
||||
: `发布目标已保存:${where};但当前核对不了生产端(${next.production_message}),下次发布前请确认。`
|
||||
? `文档级发布目标已更新:${where}。只影响单篇文档发布;整库发布写的是上面那份「整库发布目标」。`
|
||||
: `文档级发布目标已保存:${where};但当前核对不了生产端(${next.production_message}),下次发布前请确认。`
|
||||
)
|
||||
} catch (cause) {
|
||||
setError(cause instanceof Error ? cause.message : '保存生产知识库绑定失败')
|
||||
@@ -66,89 +240,23 @@ export function ProductionBindingsSection() {
|
||||
}
|
||||
}
|
||||
|
||||
const optionLabel = (base: KnowledgeBaseOption) => `${base.name}(${base.knowledge_count} 文档)`
|
||||
const renderSide = (label: string, boundID: string, boundName: string) => (
|
||||
<div className={`kb-binding-side ${boundID ? 'bound' : 'warn'}`}>
|
||||
<b>{label}</b>
|
||||
<span>
|
||||
{boundID ? boundName || `${boundID.slice(0, 8)}…` : '未绑定 · 按同 ID 语义发布(目标端须存在同 ID 库)'}
|
||||
</span>
|
||||
<small>{boundID ? 'UI 绑定 · 发布写入此库' : '未绑定:沿用测试库 ID'}</small>
|
||||
</div>
|
||||
)
|
||||
const editorFor = (value: string, onChange: (next: string) => void, placeholder: string) =>
|
||||
options.length > 0 ? (
|
||||
<select value={value} onChange={(event) => onChange(event.target.value)} aria-label={placeholder}>
|
||||
<option value="">{placeholder}</option>
|
||||
{options.map((base) => (
|
||||
<option key={base.id} value={base.id}>
|
||||
{optionLabel(base)}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
) : (
|
||||
<input
|
||||
value={value}
|
||||
onChange={(event) => onChange(event.target.value.trim())}
|
||||
placeholder={`${placeholder}(输入生产知识库 ID)`}
|
||||
aria-label={placeholder}
|
||||
/>
|
||||
)
|
||||
|
||||
return (
|
||||
<section className="settings-panel wide" id="settings-production-bindings">
|
||||
<div className="settings-section-heading">
|
||||
<div>
|
||||
<h2>生产知识库绑定</h2>
|
||||
<p>
|
||||
发布链路的“源库 → 目标库”:新发布的对内 / 对外内容按这里的绑定写入生产 WeKnora
|
||||
的对应知识库。留空表示未绑定,按同 ID 语义发布。
|
||||
</p>
|
||||
</div>
|
||||
<button onClick={() => void load()} disabled={loading}>
|
||||
{loading ? '读取中…' : '重新读取'}
|
||||
</button>
|
||||
</div>
|
||||
{error ? (
|
||||
<p className="error-message" role="alert">
|
||||
{error}
|
||||
</p>
|
||||
) : null}
|
||||
{bindings ? (
|
||||
<div className="kb-binding-grid">
|
||||
{renderSide('对内生产知识库', bindings.internal_kb_id, bindings.internal_kb_name)}
|
||||
{renderSide('对外生产知识库', bindings.external_kb_id, bindings.external_kb_name)}
|
||||
</div>
|
||||
) : null}
|
||||
{bindings && !bindings.production_connected ? (
|
||||
<p className="hint">
|
||||
生产端核对不可用:{bindings.production_message}。
|
||||
{bindings.production_base_url ? `(生产地址:${bindings.production_base_url})` : ''}
|
||||
</p>
|
||||
) : null}
|
||||
{bindings && bindings.production_connected && bindings.production_base_url ? (
|
||||
<p className="hint">
|
||||
生产端:{bindings.production_base_url}
|
||||
{bindings.updated_by ? ` · 最近修改:${bindings.updated_by}` : ''}
|
||||
</p>
|
||||
) : null}
|
||||
{isAdmin && bindings ? (
|
||||
<div className="kb-binding-editor">
|
||||
<label>对内目标库 {editorFor(internalDraft, setInternalDraft, '选择对内生产知识库')}</label>
|
||||
<label>对外目标库 {editorFor(externalDraft, setExternalDraft, '选择对外生产知识库')}</label>
|
||||
<p className="hint">保存时会向生产端核对所选的库是否真实存在;对不上会拒收,生产端不可达则只提示。</p>
|
||||
{sameSelection ? <p className="error-message">对内与对外必须绑定不同的生产知识库</p> : null}
|
||||
<button onClick={() => void save()} disabled={saving || !dirty || sameSelection}>
|
||||
{saving ? '正在核对并保存…' : '保存发布目标'}
|
||||
</button>
|
||||
{message ? (
|
||||
<p className="success-message" role="status">
|
||||
{message}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
{!isAdmin ? <p className="hint">仅管理员可修改发布目标。</p> : null}
|
||||
</section>
|
||||
<ProductionBindingsView
|
||||
bindings={bindings}
|
||||
options={options}
|
||||
internalDraft={internalDraft}
|
||||
externalDraft={externalDraft}
|
||||
isAdmin={isAdmin}
|
||||
loading={loading}
|
||||
saving={saving}
|
||||
error={error}
|
||||
message={message}
|
||||
dirty={dirty}
|
||||
sameSelection={sameSelection}
|
||||
onInternalChange={setInternalDraft}
|
||||
onExternalChange={setExternalDraft}
|
||||
onSave={() => void save()}
|
||||
onReload={() => void load()}
|
||||
/>
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
} from './librarySnapshots'
|
||||
import { useOptionalAuth } from './auth'
|
||||
import './releaseTargets.css'
|
||||
import { RestoreVerificationSection } from './RestoreVerificationSection'
|
||||
|
||||
// 发布管理:按已确认稿实现——顶部两个知识库页签,下面是**一个**作业区。
|
||||
//
|
||||
@@ -260,6 +261,7 @@ export function jobKindText(kind?: string): string {
|
||||
if (kind === 'sync') return '生产同步'
|
||||
if (kind === 'restore') return '按版本恢复'
|
||||
if (kind === 'baseline') return '建立目标基线'
|
||||
if (kind === 'verify') return '恢复验证'
|
||||
return kind ? kind : '—'
|
||||
}
|
||||
|
||||
@@ -281,7 +283,9 @@ export function publishStateText(job?: LibraryJobRecord): string {
|
||||
? '恢复进行中(目标在维护中)'
|
||||
: job.kind === 'baseline'
|
||||
? '建立基线进行中(只读捕获目标,不写内容)'
|
||||
: '同步进行中(目标在维护中)'
|
||||
: job.kind === 'verify'
|
||||
? '恢复验证进行中(目标在维护中)'
|
||||
: '同步进行中(目标在维护中)'
|
||||
}
|
||||
if (job.state === 'succeeded') return `${jobKindText(job.kind)}已完成 · ${stamp(job.finished_at ?? job.updated_at)}`
|
||||
if (job.state === 'failed') return `${jobKindText(job.kind)}失败 · ${stamp(job.finished_at ?? job.updated_at)}`
|
||||
@@ -718,6 +722,7 @@ function ProtectionCard({
|
||||
targetSealed,
|
||||
sync,
|
||||
baseline,
|
||||
scope,
|
||||
onOpen,
|
||||
}: {
|
||||
production: LibraryOverview['production'] | null
|
||||
@@ -730,6 +735,7 @@ function ProtectionCard({
|
||||
targetSealed: boolean
|
||||
sync: ReleaseSyncActions
|
||||
baseline: ReleaseBaselineActions
|
||||
scope: Scope
|
||||
onOpen: () => void
|
||||
}) {
|
||||
const backupKnown = prebackup.state === 'known' && prebackup.version
|
||||
@@ -798,6 +804,7 @@ function ProtectionCard({
|
||||
<dt>恢复条件</dt>
|
||||
<dd>{gate.reasons.length === 0 ? '已满足' : `缺 ${gate.reasons.length} 项`}</dd>
|
||||
</dl>
|
||||
<RestoreVerificationSection scope={scope} isAdmin={sync.canSign} />
|
||||
{gate.reasons.length > 0 ? (
|
||||
<ul className="rel-gate" aria-label="恢复与启动条件">
|
||||
{gate.reasons.map((reason) => (
|
||||
@@ -880,6 +887,7 @@ function ReadyBody({
|
||||
targetSealed={Boolean(snapshot)}
|
||||
sync={sync}
|
||||
baseline={baseline}
|
||||
scope={view.scope}
|
||||
onOpen={onOpen}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import { renderToStaticMarkup } from 'react-dom/server'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { RestoreVerificationView } from './RestoreVerificationSection'
|
||||
import type { LibraryVerificationView } from './restoreVerification'
|
||||
|
||||
const target = {
|
||||
instance: 'weknora-test',
|
||||
project: 'weknora',
|
||||
internal_kb: 'a8e9c8ad-411c-4c53-8421-599d29e9b179',
|
||||
external_kb: '2e980686-2233-4c4e-b5a6-8b0069477b8e',
|
||||
index: 'weknora_test_chunks',
|
||||
s3_bucket: 'dobot-weknora-test',
|
||||
tenant_id: 10000,
|
||||
database_configured: true,
|
||||
}
|
||||
|
||||
const view = (overrides: Partial<LibraryVerificationView> = {}): LibraryVerificationView => ({
|
||||
scope: 'internal',
|
||||
target,
|
||||
verified: false,
|
||||
reason: '还没有对当前目标做过恢复验证',
|
||||
latest: null,
|
||||
history: [],
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const render = (value: LibraryVerificationView | null, isAdmin = true, pending = false, error: string | null = null) =>
|
||||
renderToStaticMarkup(
|
||||
<RestoreVerificationView
|
||||
view={value}
|
||||
scope="internal"
|
||||
isAdmin={isAdmin}
|
||||
pending={pending}
|
||||
error={error}
|
||||
notice=""
|
||||
onRun={() => undefined}
|
||||
onReload={() => undefined}
|
||||
/>
|
||||
)
|
||||
|
||||
describe('RestoreVerificationView', () => {
|
||||
it('没有证据时显示"还没有验证记录"并给出下一步,不显示已验证', () => {
|
||||
const html = render(view())
|
||||
expect(html).toContain('还没有验证记录')
|
||||
expect(html).toContain('跑一次恢复验证')
|
||||
expect(html).not.toContain('已验证(当前目标)')
|
||||
})
|
||||
|
||||
it('管理员能看到发起按钮,工程师只看到证据与原因', () => {
|
||||
expect(render(view(), true)).toContain('跑一次恢复验证')
|
||||
const engineer = render(view(), false)
|
||||
expect(engineer).not.toContain('跑一次恢复验证')
|
||||
expect(engineer).toContain('仅管理员可发起')
|
||||
})
|
||||
|
||||
it('通过后显示已验证、快照与完整目标', () => {
|
||||
const html = render(
|
||||
view({
|
||||
verified: true,
|
||||
reason: '',
|
||||
latest: {
|
||||
id: 'libverify-internal-job1',
|
||||
scope: 'internal',
|
||||
status: 'passed',
|
||||
snapshot_id: '0f9b39a16738',
|
||||
job_id: 'libverify-internal-1',
|
||||
started_at: '2026-11-03T01:00:00Z',
|
||||
finished_at: '2026-11-03T01:02:00Z',
|
||||
},
|
||||
})
|
||||
)
|
||||
expect(html).toContain('已验证(当前目标)')
|
||||
expect(html).toContain('0f9b39a16738')
|
||||
expect(html).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179')
|
||||
expect(html).toContain('weknora_test_chunks')
|
||||
})
|
||||
|
||||
it('失败时把失败原因写出来,按钮仍可重试', () => {
|
||||
const html = render(
|
||||
view({
|
||||
latest: {
|
||||
id: 'libverify-internal-job2',
|
||||
scope: 'internal',
|
||||
status: 'failed',
|
||||
snapshot_id: 'snap-2',
|
||||
job_id: 'libverify-internal-2',
|
||||
failure: '恢复后内容仍与预备份不一致',
|
||||
started_at: '2026-11-03T01:00:00Z',
|
||||
finished_at: '2026-11-03T01:05:00Z',
|
||||
},
|
||||
})
|
||||
)
|
||||
expect(html).toContain('最近一次未通过')
|
||||
expect(html).toContain('恢复后内容仍与预备份不一致')
|
||||
expect(html).toContain('跑一次恢复验证')
|
||||
})
|
||||
|
||||
it('投递被拒时显示后端逐条原因', () => {
|
||||
const html = render(view(), true, false, '恢复验证被拒绝:生产写入闸口关闭')
|
||||
expect(html).toContain('生产写入闸口关闭')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,140 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
|
||||
import {
|
||||
enqueueLibraryVerification,
|
||||
fetchLibraryVerification,
|
||||
verificationStatusText,
|
||||
verificationTargetLines,
|
||||
} from './restoreVerification'
|
||||
import type { LibraryVerificationView } from './restoreVerification'
|
||||
|
||||
// 「恢复验证」面板:同步闸口认的"生产可恢复"证据只能由这里发起的一次真实演练产生。
|
||||
//
|
||||
// 为什么要有这个入口:以前只有 CLI(现为 internal/libraryverify 的薄封装)能跑演练,
|
||||
// 页面只能看到一个 env 开关。运维在页面上看到"缺 1 项",却没有任何可点的下一步。
|
||||
// 这里把"跑一次真实演练"变成正常流程:它会停写入者、写 canary、按预备份恢复,
|
||||
// 全过程由 worker 执行并登记真实终态;页面只显示结果,不提供手填开关。
|
||||
|
||||
export interface RestoreVerificationViewProps {
|
||||
view: LibraryVerificationView | null
|
||||
scope: string
|
||||
isAdmin: boolean
|
||||
pending: boolean
|
||||
error: string | null
|
||||
notice: string
|
||||
onRun: () => void
|
||||
onReload: () => void
|
||||
}
|
||||
|
||||
const stampText = (value: string): string => {
|
||||
if (!value) return '—'
|
||||
const parsed = new Date(value)
|
||||
return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString()
|
||||
}
|
||||
|
||||
export function RestoreVerificationView(props: RestoreVerificationViewProps) {
|
||||
const { view, scope, isAdmin, pending, error, notice, onRun, onReload } = props
|
||||
const status = verificationStatusText(view)
|
||||
const latest = view?.latest ?? null
|
||||
const running = Boolean(view?.running)
|
||||
const tone = view?.verified ? 'green' : latest?.status === 'failed' ? 'red' : 'gray'
|
||||
return (
|
||||
<section className="rel-verification" aria-label="恢复验证">
|
||||
<header className="rel-between">
|
||||
<h3>恢复验证(当前整库目标)</h3>
|
||||
<span className={`rel-pill ${tone}`}>{status}</span>
|
||||
</header>
|
||||
{view ? (
|
||||
<ul className="rel-target-lines" aria-label="恢复验证目标">
|
||||
{verificationTargetLines(view.target, scope).map((line) => (
|
||||
<li key={line}>{line}</li>
|
||||
))}
|
||||
</ul>
|
||||
) : null}
|
||||
<p className="rel-hint">{view?.reason || '还没有读到验证状态。'}</p>
|
||||
{latest ? (
|
||||
<p className="rel-hint">
|
||||
最近一次:{latest.status === 'passed' ? '通过' : '未通过'} · 快照 {latest.snapshot_id.slice(0, 12) || '—'} ·{' '}
|
||||
{stampText(latest.finished_at)} · 作业 {latest.job_id}
|
||||
</p>
|
||||
) : null}
|
||||
{latest?.failure ? <p className="rel-warn">未通过原因:{latest.failure}</p> : null}
|
||||
{error ? <p className="rel-warn">{error}</p> : null}
|
||||
{notice ? <p className="rel-hint">{notice}</p> : null}
|
||||
<div className="rel-drawer-actions">
|
||||
{isAdmin ? (
|
||||
<button type="button" className="rel-secondary" onClick={onRun} disabled={pending || running}>
|
||||
{pending ? '投递中…' : running ? '验证执行中…' : '跑一次恢复验证(维护隔离)'}
|
||||
</button>
|
||||
) : (
|
||||
<span className="rel-hint">恢复验证仅管理员可发起;工程师可以看到证据与原因。</span>
|
||||
)}
|
||||
<button type="button" className="rel-secondary" onClick={onReload} disabled={pending}>
|
||||
刷新验证状态
|
||||
</button>
|
||||
</div>
|
||||
<p className="rel-hint">
|
||||
验证会停住目标写入者、写一个隔离 canary 证明校验有效、再按预备份恢复并逐类复核;
|
||||
通过后证据按当前目标登记,换库/换索引/换桶后自动失效。
|
||||
</p>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
// 容器:挂载即读一次,之后每 5 秒刷新(验证作业在 worker 里跑,页面只读证据)。
|
||||
export function RestoreVerificationSection({ scope, isAdmin }: { scope: string; isAdmin: boolean }) {
|
||||
const [view, setView] = useState<LibraryVerificationView | null>(null)
|
||||
const [pending, setPending] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [notice, setNotice] = useState('')
|
||||
const mounted = useRef(true)
|
||||
|
||||
const reload = useCallback(
|
||||
(signal?: AbortSignal) => {
|
||||
fetchLibraryVerification(scope, signal)
|
||||
.then((next) => {
|
||||
if (mounted.current && next) setView(next)
|
||||
})
|
||||
.catch(() => undefined)
|
||||
},
|
||||
[scope]
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
mounted.current = true
|
||||
const controller = new AbortController()
|
||||
reload(controller.signal)
|
||||
const timer = setInterval(() => reload(controller.signal), 5000)
|
||||
return () => {
|
||||
mounted.current = false
|
||||
controller.abort()
|
||||
clearInterval(timer)
|
||||
}
|
||||
}, [reload])
|
||||
|
||||
const onRun = useCallback(() => {
|
||||
setPending(true)
|
||||
setError(null)
|
||||
setNotice('')
|
||||
enqueueLibraryVerification(scope)
|
||||
.then((accepted) => {
|
||||
setNotice(`已投递恢复验证作业 ${accepted.job_id};进度在作业进度里可见,完成后这里显示证据。`)
|
||||
reload()
|
||||
})
|
||||
.catch((err: unknown) => setError(err instanceof Error ? err.message : '投递恢复验证失败'))
|
||||
.finally(() => setPending(false))
|
||||
}, [scope, reload])
|
||||
|
||||
return (
|
||||
<RestoreVerificationView
|
||||
view={view}
|
||||
scope={scope}
|
||||
isAdmin={isAdmin}
|
||||
pending={pending}
|
||||
error={error}
|
||||
notice={notice}
|
||||
onRun={onRun}
|
||||
onReload={() => reload()}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -332,7 +332,8 @@ describe('SystemSettingsView', () => {
|
||||
// 发布目标(源库 → 目标库)是独立分区,与测试端收件库分开。
|
||||
const target = view('production-bindings')
|
||||
expect(target).toContain('id="settings-production-bindings"')
|
||||
expect(target).toContain('新发布的对内 / 对外内容按这里的绑定写入生产 WeKnora 的对应知识库')
|
||||
expect(target).toContain('整库发布目标')
|
||||
expect(target).toContain('文档级绑定')
|
||||
})
|
||||
it('does not show healthy simulation when the settings data failed to load', () => {
|
||||
const html = renderToStaticMarkup(
|
||||
|
||||
@@ -12,6 +12,8 @@ export interface BackupSchedule {
|
||||
last_applied_summary: string
|
||||
host_configured: boolean
|
||||
host_target: string
|
||||
// host_endpoint 是这条别名真实拨号的 host:port(来自挂载的 ssh 配置)。
|
||||
host_endpoint: string
|
||||
host_reachable: boolean
|
||||
host_error: string
|
||||
host_schedule_enabled: boolean
|
||||
@@ -70,6 +72,7 @@ function scheduleProjection(item: Record<string, unknown>): BackupSchedule {
|
||||
last_applied_summary: asString(item.last_applied_summary),
|
||||
host_configured: item.host_configured === true,
|
||||
host_target: asString(item.host_target),
|
||||
host_endpoint: asString(item.host_endpoint),
|
||||
host_reachable: item.host_reachable === true,
|
||||
host_error: asString(item.host_error),
|
||||
host_schedule_enabled: item.host_schedule_enabled === true,
|
||||
|
||||
@@ -567,7 +567,7 @@ export interface LibraryStageRow {
|
||||
export interface LibraryJobRecord {
|
||||
id: string
|
||||
scope: string
|
||||
kind: 'seal' | 'sync' | 'restore' | 'baseline'
|
||||
kind: 'seal' | 'sync' | 'restore' | 'baseline' | 'verify'
|
||||
state: 'pending' | 'running' | 'succeeded' | 'failed' | 'cancelled'
|
||||
stage: string
|
||||
request_digest: string
|
||||
|
||||
@@ -11,6 +11,22 @@ export interface ProductionBindings {
|
||||
production_base_url: string
|
||||
production_connected: boolean
|
||||
production_message: string
|
||||
// legacy_scope 标明这条记录属于旧文档级发布路径;整库发布不看它。
|
||||
legacy_scope: string
|
||||
// whole_target 是整库发布真正写入的目标(来自部署配置),只读。
|
||||
whole_target: LibraryPublishTarget
|
||||
}
|
||||
|
||||
export interface LibraryPublishTarget {
|
||||
configured: boolean
|
||||
source: string
|
||||
tenant_id: number
|
||||
internal_kb_id: string
|
||||
external_kb_id: string
|
||||
database: string
|
||||
elastic_index: string
|
||||
object_bucket: string
|
||||
missing_reason: string
|
||||
}
|
||||
|
||||
const record = (value: unknown): value is Record<string, unknown> =>
|
||||
@@ -26,6 +42,24 @@ function projection(item: Record<string, unknown>): ProductionBindings {
|
||||
production_base_url: typeof item.production_base_url === 'string' ? item.production_base_url : '',
|
||||
production_connected: item.production_connected === true,
|
||||
production_message: typeof item.production_message === 'string' ? item.production_message : '',
|
||||
legacy_scope: typeof item.legacy_scope === 'string' ? item.legacy_scope : '',
|
||||
whole_target: wholeTarget(item.whole_target),
|
||||
}
|
||||
}
|
||||
|
||||
// wholeTarget 只读投影:缺字段就说未配置,不拿可编辑的文档级绑定冒充整库目标。
|
||||
function wholeTarget(value: unknown): LibraryPublishTarget {
|
||||
const item = record(value) ? value : {}
|
||||
return {
|
||||
configured: item.configured === true,
|
||||
source: typeof item.source === 'string' ? item.source : '',
|
||||
tenant_id: typeof item.tenant_id === 'number' ? item.tenant_id : 0,
|
||||
internal_kb_id: typeof item.internal_kb_id === 'string' ? item.internal_kb_id : '',
|
||||
external_kb_id: typeof item.external_kb_id === 'string' ? item.external_kb_id : '',
|
||||
database: typeof item.database === 'string' ? item.database : '',
|
||||
elastic_index: typeof item.elastic_index === 'string' ? item.elastic_index : '',
|
||||
object_bucket: typeof item.object_bucket === 'string' ? item.object_bucket : '',
|
||||
missing_reason: typeof item.missing_reason === 'string' ? item.missing_reason : '',
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
parseVerificationRecord,
|
||||
parseVerificationView,
|
||||
verificationStatusText,
|
||||
verificationTargetLines,
|
||||
} from './restoreVerification'
|
||||
import type { LibraryVerificationView } from './restoreVerification'
|
||||
|
||||
const target = {
|
||||
instance: 'weknora-test',
|
||||
project: 'weknora',
|
||||
internal_kb: 'a8e9c8ad-411c-4c53-8421-599d29e9b179',
|
||||
external_kb: '2e980686-2233-4c4e-b5a6-8b0069477b8e',
|
||||
index: 'weknora_test_chunks',
|
||||
s3_bucket: 'dobot-weknora-test',
|
||||
tenant_id: 10000,
|
||||
database_configured: true,
|
||||
}
|
||||
|
||||
const passed = {
|
||||
id: 'libverify-internal-job1',
|
||||
scope: 'internal',
|
||||
status: 'passed',
|
||||
snapshot_id: '0f9b39a16738',
|
||||
job_id: 'libverify-internal-1',
|
||||
started_at: '2026-11-03T01:00:00Z',
|
||||
finished_at: '2026-11-03T01:02:00Z',
|
||||
}
|
||||
|
||||
describe('restoreVerification parsing', () => {
|
||||
it('只把后端明确 true 当作已验证', () => {
|
||||
const view = parseVerificationView({ scope: 'internal', target, verified: 'true', reason: '', history: [] })
|
||||
expect(view?.verified).toBe(false)
|
||||
})
|
||||
|
||||
it('保留失败证据的失败原因', () => {
|
||||
const record = parseVerificationRecord({ ...passed, status: 'failed', failure: '恢复后内容仍与预备份不一致' })
|
||||
expect(record?.status).toBe('failed')
|
||||
expect(record?.failure).toBe('恢复后内容仍与预备份不一致')
|
||||
})
|
||||
|
||||
it('未知状态一律丢弃,不猜成通过', () => {
|
||||
expect(parseVerificationRecord({ ...passed, status: 'running' })).toBeNull()
|
||||
})
|
||||
|
||||
it('没有目标信息的响应视为无效', () => {
|
||||
expect(parseVerificationView({ scope: 'internal', verified: true })).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('restoreVerification display', () => {
|
||||
const base: LibraryVerificationView = {
|
||||
scope: 'internal',
|
||||
target,
|
||||
verified: true,
|
||||
reason: '',
|
||||
latest: {
|
||||
id: passed.id,
|
||||
scope: 'internal',
|
||||
status: 'passed',
|
||||
snapshot_id: passed.snapshot_id,
|
||||
job_id: passed.job_id,
|
||||
started_at: passed.started_at,
|
||||
finished_at: passed.finished_at,
|
||||
},
|
||||
history: [],
|
||||
}
|
||||
|
||||
it('整库目标显示完整 KB ID 与索引/桶,不缩写成"2074 文档库"', () => {
|
||||
const lines = verificationTargetLines(target, 'internal')
|
||||
expect(lines.join(' | ')).toContain('a8e9c8ad-411c-4c53-8421-599d29e9b179')
|
||||
expect(lines.join(' | ')).toContain('weknora_test_chunks')
|
||||
expect(lines.join(' | ')).toContain('dobot-weknora-test')
|
||||
})
|
||||
|
||||
it('对外 scope 显示对外目标', () => {
|
||||
expect(verificationTargetLines(target, 'external').join(' | ')).toContain('2e980686-2233-4c4e-b5a6-8b0069477b8e')
|
||||
})
|
||||
|
||||
it('状态文案区分已验证 / 最近失败 / 无记录', () => {
|
||||
expect(verificationStatusText(base)).toBe('已验证(当前目标)')
|
||||
expect(
|
||||
verificationStatusText({
|
||||
...base,
|
||||
verified: false,
|
||||
latest: { ...base.latest!, status: 'failed' },
|
||||
})
|
||||
).toBe('最近一次未通过')
|
||||
expect(verificationStatusText({ ...base, verified: false, latest: null })).toBe('还没有验证记录')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,158 @@
|
||||
// 恢复验证的页面数据层:读证据 + 投递真实演练。
|
||||
//
|
||||
// 关键约定:verified 只由后端证据回答(当前目标 + scope 的最近一次真实演练)。
|
||||
// 页面不提供任何"我确认已验证"的开关——那正是旧实现的问题(一个可手填的 env)。
|
||||
export type LibraryVerificationStatus = 'passed' | 'failed'
|
||||
|
||||
export interface LibraryVerificationRecord {
|
||||
id: string
|
||||
scope: string
|
||||
status: LibraryVerificationStatus
|
||||
snapshot_id: string
|
||||
job_id: string
|
||||
failure?: string
|
||||
started_at: string
|
||||
finished_at: string
|
||||
}
|
||||
|
||||
export interface LibraryVerificationTarget {
|
||||
instance: string
|
||||
project: string
|
||||
internal_kb: string
|
||||
external_kb: string
|
||||
index: string
|
||||
s3_bucket: string
|
||||
tenant_id: number
|
||||
database_configured: boolean
|
||||
}
|
||||
|
||||
export interface LibraryVerificationView {
|
||||
scope: string
|
||||
target: LibraryVerificationTarget
|
||||
verified: boolean
|
||||
reason: string
|
||||
latest?: LibraryVerificationRecord | null
|
||||
history: LibraryVerificationRecord[]
|
||||
running?: boolean
|
||||
job_id?: string
|
||||
}
|
||||
|
||||
function record(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null
|
||||
}
|
||||
|
||||
function text(value: unknown): string {
|
||||
return typeof value === 'string' ? value : ''
|
||||
}
|
||||
|
||||
function bool(value: unknown): boolean {
|
||||
return value === true
|
||||
}
|
||||
|
||||
export function parseVerificationTarget(value: unknown): LibraryVerificationTarget | null {
|
||||
if (!record(value)) return null
|
||||
return {
|
||||
instance: text(value.instance),
|
||||
project: text(value.project),
|
||||
internal_kb: text(value.internal_kb),
|
||||
external_kb: text(value.external_kb),
|
||||
index: text(value.index),
|
||||
s3_bucket: text(value.s3_bucket),
|
||||
tenant_id: typeof value.tenant_id === 'number' ? value.tenant_id : 0,
|
||||
database_configured: bool(value.database_configured),
|
||||
}
|
||||
}
|
||||
|
||||
export function parseVerificationRecord(value: unknown): LibraryVerificationRecord | null {
|
||||
if (!record(value)) return null
|
||||
const status = text(value.status)
|
||||
if (status !== 'passed' && status !== 'failed') return null
|
||||
return {
|
||||
id: text(value.id),
|
||||
scope: text(value.scope),
|
||||
status,
|
||||
snapshot_id: text(value.snapshot_id),
|
||||
job_id: text(value.job_id),
|
||||
failure: text(value.failure) || undefined,
|
||||
started_at: text(value.started_at),
|
||||
finished_at: text(value.finished_at),
|
||||
}
|
||||
}
|
||||
|
||||
export function parseVerificationView(value: unknown): LibraryVerificationView | null {
|
||||
if (!record(value)) return null
|
||||
const target = parseVerificationTarget(value.target)
|
||||
if (!target) return null
|
||||
const history = Array.isArray(value.history)
|
||||
? value.history.map(parseVerificationRecord).filter((item): item is LibraryVerificationRecord => item !== null)
|
||||
: []
|
||||
const latest = parseVerificationRecord(value.latest) ?? null
|
||||
return {
|
||||
scope: text(value.scope),
|
||||
target,
|
||||
// 只有后端明确说 true 才算已验证:任何解析失败都退回"没有证据"。
|
||||
verified: bool(value.verified),
|
||||
reason: text(value.reason),
|
||||
latest,
|
||||
history,
|
||||
running: bool(value.running),
|
||||
job_id: text(value.job_id) || undefined,
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchLibraryVerification(
|
||||
scope: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<LibraryVerificationView | null> {
|
||||
const response = await fetch(`/api/v1/library/verifications?scope=${encodeURIComponent(scope)}`, {
|
||||
headers: { Accept: 'application/json' },
|
||||
signal,
|
||||
})
|
||||
const value = (await response.json().catch(() => null)) as unknown
|
||||
return parseVerificationView(value)
|
||||
}
|
||||
|
||||
export interface LibraryVerificationAccepted {
|
||||
job_id: string
|
||||
scope: string
|
||||
}
|
||||
|
||||
// 投递一次真实恢复验证。失败时把后端的逐条拒绝原因原样显示,
|
||||
// 不把它包装成"稍后再试"——否则用户不知道是闸口、维护还是目标不可达。
|
||||
export async function enqueueLibraryVerification(scope: string): Promise<LibraryVerificationAccepted> {
|
||||
const response = await fetch('/api/v1/library/verifications', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
|
||||
body: JSON.stringify({ scope }),
|
||||
})
|
||||
const value = (await response.json().catch(() => null)) as unknown
|
||||
if (!response.ok) {
|
||||
if (response.status === 403) throw new Error('恢复验证仅管理员可发起。')
|
||||
if (record(value) && Array.isArray(value.reasons)) {
|
||||
const reasons = value.reasons.map(text).filter(Boolean)
|
||||
if (reasons.length > 0) throw new Error(`恢复验证被拒绝:${reasons.join(';')}`)
|
||||
}
|
||||
if (record(value) && text(value.error)) throw new Error(text(value.error))
|
||||
throw new Error(`投递恢复验证失败(HTTP ${response.status})`)
|
||||
}
|
||||
if (!record(value) || !record(value.job)) throw new Error('恢复验证作业响应无效')
|
||||
return { job_id: text(value.job.id), scope: text(value.job.scope) }
|
||||
}
|
||||
|
||||
// 目标展示:整库目标必须显示完整 ID 与端点,避免"看着像 2074 文档库"这类误读。
|
||||
export function verificationTargetLines(target: LibraryVerificationTarget, scope: string): string[] {
|
||||
const kb = scope === 'internal' ? target.internal_kb : target.external_kb
|
||||
return [
|
||||
`实例 ${target.instance || '未配置'} · 项目 ${target.project || '未配置'}`,
|
||||
`知识库 ${kb || '未配置'}`,
|
||||
`索引 ${target.index || '未配置'} · 桶 ${target.s3_bucket || '未配置'}`,
|
||||
`租户 ${target.tenant_id || '未配置'} · 数据库${target.database_configured ? '已配置' : '未配置'}`,
|
||||
]
|
||||
}
|
||||
|
||||
export function verificationStatusText(view: LibraryVerificationView | null): string {
|
||||
if (!view) return '还没有读到验证状态'
|
||||
if (view.verified) return '已验证(当前目标)'
|
||||
if (view.latest) return view.latest.status === 'failed' ? '最近一次未通过' : '未通过'
|
||||
return '还没有验证记录'
|
||||
}
|
||||
Reference in New Issue
Block a user