feat(library): 恢复验证成为可页面发起的真实作业
build / build (push) Successful in 1m40s

根因:生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关,
页面没有任何入口去真跑一次恢复;CLI(librarydrill)又不是 worker 能复用的实现,
所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。

改动:
- 共享实现 internal/libraryverify(维护隔离→捕获→基线校验→canary→恢复→复核→关窗口),
  cmd/librarydrill 改为薄封装,CLI 与 worker 只允许一份执行体。
- verify 成为独立业务作业:kind + VerifyStages(capture/verify/canary/restore/finalize),
  与 sync/restore/baseline 共用同一把写互斥;迁移 79 加 kind 约束、重建部分唯一索引、
  新建 library_restore_verifications(按 target_id 绑定,换库/索引/桶旧证据自动失效,
  同 target+scope+job 幂等)。
- worker:admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现,阶段实时写
  library_job_stages,结束后写真实终态;通过写 passed 证据,失败写 failed+原因(不伪成功);
  中断接管按同一作业 id 收掉维护窗口。
- API:POST/GET /api/v1/library/verifications(管理员;闸口关闭/维护中拒绝;返回完整
  整库目标、最近证据、失败原因),OpenAPI 同步。
- 闸口:同步/恢复/续作的证据判断改为读当前目标证据;历史 env 仅在证据缺失时兜底,
  证据存在时以证据为准。
- 页面:生产保护与恢复面板加「恢复验证」块(完整 KB/索引/桶、状态、最近证据、失败原因、
  管理员按钮、5s 轮询),作业类型补 verify 文案。

测试:make verify 16/16 全绿(go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定);
新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
This commit is contained in:
姚章浩
2026-11-03 10:14:08 +08:00
parent 4421dd38ce
commit 5e1482e349
47 changed files with 2502 additions and 353 deletions
+12 -2
View File
@@ -24,6 +24,9 @@ type BackupScheduleStore interface {
type BackupHost interface {
Available() bool
Target() string
// Endpoint 是挂载的 ssh 配置里该别名真实拨号的 host:port;页面据此证明
// "连的是哪条通道",而不是只显示一个别名。
Endpoint() string
Show(context.Context) (backuphost.ShowResult, error)
Apply(context.Context, bool, string, int, int) (backuphost.ApplyResult, error)
RunNow(context.Context) (backuphost.RunResult, error)
@@ -82,8 +85,11 @@ type backupScheduleView struct {
// HostConfigured/HostReachable 区分"没接线"与"接线了但连不上"。
HostConfigured bool `json:"host_configured"`
HostTarget string `json:"host_target"`
HostReachable bool `json:"host_reachable"`
HostError string `json:"host_error"`
// HostEndpoint 是这条别名真实拨号的 host:port(来自挂载的 ssh 配置)。
// 连接失败时它就是"到底往哪连"的唯一证据,不能只给一个别名让人猜。
HostEndpoint string `json:"host_endpoint"`
HostReachable bool `json:"host_reachable"`
HostError string `json:"host_error"`
HostScheduleEnabled bool `json:"host_schedule_enabled"`
HostScheduleTime string `json:"host_schedule_time"`
@@ -281,6 +287,7 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
}
view.HostConfigured = config.BackupHost.Available()
view.HostTarget = config.BackupHost.Target()
view.HostEndpoint = config.BackupHost.Endpoint()
if !view.HostConfigured {
view.InSync = false
view.HostError = "目标主机凭据未挂载(BACKUP_HOST_SSH_CONFIG / BACKUP_HOST_TARGET)"
@@ -293,6 +300,9 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
if err != nil {
view.HostReachable = false
view.HostError = err.Error()
if view.HostEndpoint != "" {
view.HostError += "(拨号地址 " + view.HostEndpoint + ")"
}
view.InSync = false
} else {
view.HostReachable = true
+46
View File
@@ -46,6 +46,7 @@ func (store *fakeBackupScheduleStore) MarkBackupScheduleApplied(_ context.Contex
type fakeBackupHost struct {
available bool
target string
endpoint string
show backuphost.ShowResult
showErr error
apply backuphost.ApplyResult
@@ -59,6 +60,9 @@ type fakeBackupHost struct {
func (host *fakeBackupHost) Available() bool { return host.available }
func (host *fakeBackupHost) Target() string { return host.target }
func (host *fakeBackupHost) Endpoint() string {
return host.endpoint
}
func (host *fakeBackupHost) Show(context.Context) (backuphost.ShowResult, error) {
return host.show, host.showErr
}
@@ -166,6 +170,48 @@ func TestHandleBackupScheduleHostFlagsDrift(t *testing.T) {
}
}
func TestHandleBackupScheduleHostUnreachableSeparatesConnectionFromHostState(t *testing.T) {
// 现场原错:ssh connect to host 120.76.217.51 port 22 Operation timed out。
// 连接没建立 ⇒ 既不能报"脚本缺失",也不能报"计划未下发",只能报不可达 + 拨号地址。
host := &fakeBackupHost{
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
showErr: errors.New("目标主机 weknora-test 不可达(ssh: connect to host 120.76.217.51 port 22: Operation timed out);未执行任何主机侧脚本,脚本与计划状态未知"),
}
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
recorder := httptest.NewRecorder()
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
data := decodeData(t, recorder)
if data["host_reachable"] != false || data["in_sync"] != false {
t.Fatalf("连不上时不能报告可达或已同步: %+v", data)
}
message := data["host_error"].(string)
if !strings.Contains(message, "不可达") || !strings.Contains(message, "100.85.111.121:22") {
t.Fatalf("应报不可达并给出真实拨号地址: %q", message)
}
if strings.Contains(message, "脚本缺失") || strings.Contains(message, "未下发") {
t.Fatalf("未观测到的脚本/计划状态不能当成结论: %q", message)
}
}
func TestHandleBackupScheduleHostReportsMissingScriptOnlyAfterConnecting(t *testing.T) {
// 真正的脚本缺失:ssh 已连上,主机回的是脚本不存在 ⇒ 这时才允许说脚本缺失。
host := &fakeBackupHost{
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
showErr: errors.New("已连上 weknora-test,但主机侧命令失败: bash: /opt/weknora-test/install_backup_schedule.sh: No such file or directory"),
}
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
recorder := httptest.NewRecorder()
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
data := decodeData(t, recorder)
message := data["host_error"].(string)
if !strings.Contains(message, "已连上") || !strings.Contains(message, "No such file") {
t.Fatalf("已连接但脚本缺失时应原样带回主机原因: %q", message)
}
if strings.Contains(message, "不可达") {
t.Fatalf("已连上就不能说不可达: %q", message)
}
}
func TestHandleBackupScheduleHostUnconfiguredDegrades(t *testing.T) {
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, &fakeBackupHost{available: false})
recorder := httptest.NewRecorder()
+4 -5
View File
@@ -642,9 +642,7 @@ func librarySyncGateWithGrant(config Config, production libraryKnownView, prebac
if !config.ProductionReleaseRequestsEnabled {
gate.Reasons = append(gate.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
}
if !config.ProductionBackupRestoreVerified {
gate.Reasons = append(gate.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
}
gate.Reasons = append(gate.Reasons, restoreGateReasons(config)...)
// 生产端点以"整库目标是否配置"为准:整库同步直接写 PG/ES/对象,
// 不依赖 WeKnora 的 HTTP 端,所以不能拿它是否配置当门禁。
if strings.TrimSpace(config.LibraryProductionDatabaseURL) == "" {
@@ -814,7 +812,8 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi
grant, hasGrant := activeGrantFor(r.Context(), config, scope, target)
overview.Authorization = authorizationViewOf(grant, hasGrant)
overview.Gate = librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, target, hasGrant)
// 闸口按当前目标的真实恢复证据判断(没有证据时也接受历史 env 覆盖)。
overview.Gate = librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), scope), overview.Production, overview.Prebackup, target, hasGrant)
writeJSON(w, http.StatusOK, overview)
}
@@ -963,7 +962,7 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
return
}
}
gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant)
gate := librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), request.Scope), overview.Production, overview.Prebackup, request.Target, hasGrant)
if !gate.SyncAllowed {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
"code": "library_sync_blocked", "reasons": gate.Reasons,
+3 -9
View File
@@ -71,9 +71,7 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), request.Scope))...)
authorizationOK := false
if storeWithAuth, ok := config.Store.(interface {
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
@@ -169,9 +167,7 @@ func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config C
if !config.ProductionReleaseRequestsEnabled {
view.Reasons = append(view.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
}
if !config.ProductionBackupRestoreVerified {
view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
}
view.Reasons = append(view.Reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), scope))...)
digest, err := libraryRestoreParamsDigest(scope, snapshotID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
@@ -228,9 +224,7 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
if !config.ProductionReleaseRequestsEnabled {
reasons = append(reasons, "生产写入闸口关闭")
}
if !config.ProductionBackupRestoreVerified {
reasons = append(reasons, "没有已核验的生产恢复演练证据")
}
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), source.Scope))...)
var digest string
switch source.Kind {
case governance.LibraryJobSync:
+219
View File
@@ -0,0 +1,219 @@
package httpapi
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"time"
"knowledge-governance/internal/governance"
)
// 恢复验证:页面点一次"跑恢复验证",后端建一个持久作业,worker 用与 CLI 同一份实现
// (internal/libraryverify)真实执行:捕获目标 → 基线校验 → 写 canary 证明校验有效 →
// 按预备份恢复 → 恢复后复核。通过与否只由这条证据表回答,页面不接受手填开关。
const libraryVerifyJobType = "library.verify"
type libraryVerifyRequest struct {
Scope string `json:"scope"`
}
// libraryVerificationView 是页面需要的全部事实:当前目标是谁、最近一次验证结果、
// 证据是否对当前目标成立、以及同步闸口能不能据此放行。
type libraryVerificationView struct {
Scope string `json:"scope"`
Target libraryVerificationTarget `json:"target"`
Verified bool `json:"verified"`
Reason string `json:"reason"`
Latest *governance.LibraryRestoreVerification `json:"latest,omitempty"`
History []governance.LibraryRestoreVerification `json:"history"`
Running bool `json:"running"`
JobID string `json:"job_id,omitempty"`
}
type libraryVerificationTarget struct {
Instance string `json:"instance"`
Project string `json:"project"`
InternalKB string `json:"internal_kb"`
ExternalKB string `json:"external_kb"`
Index string `json:"index"`
S3Bucket string `json:"s3_bucket"`
TenantID uint64 `json:"tenant_id"`
DatabaseSet bool `json:"database_configured"`
}
// libraryVerifyEnqueue 投递一次恢复验证作业(管理员)。
//
// 它不签发发布授权、不改版本:验证只回答"当前目标的预备份能不能恢复"。
// 目标是部署配置,不接受请求体指定——否则一次点击就能验证别的库并把证据挂到当前目标上。
func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
if r.Method != http.MethodPost {
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
return
}
identity, ok := sessionFrom(r.Context())
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
writeJSON(w, http.StatusForbidden, map[string]any{"error": "恢复验证仅管理员可发起"})
return
}
var request libraryVerifyRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
return
}
if request.Scope != "internal" && request.Scope != "external" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
return
}
store, ok := libraryStore(config)
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_verify_unavailable",
"error": "恢复验证存储不可用"})
return
}
// 验证要停写入者并在目标上写 canary:维护中或闸口关闭时直接拒绝,
// 不让页面投一个 worker 必然失败的作业。
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
writeJSON(w, http.StatusConflict, body)
return
}
if !config.ProductionReleaseRequestsEnabled {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
"code": "library_verify_blocked",
"reasons": []string{
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary",
},
})
return
}
digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobVerify,
map[string]any{"scope": request.Scope})
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"})
return
}
now := time.Now().UTC()
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
Kind: governance.LibraryJobVerify,
IdempotencyKey: "library.verify:" + request.Scope + ":" + now.Format("20060102T150405.000000000"),
RequestDigest: digest, Actor: identity.Username,
}, governance.VerifyStages(), false, now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active",
"error": "该目标已有在飞的写作业(同步/恢复/基线/验证互斥),等它结束再发起"})
return
}
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
"error": "同一幂等键对应了不同的请求"})
return
}
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": reused})
}
// libraryVerifications 读当前目标的恢复验证证据(页面轮询用)。
func libraryVerifications(w http.ResponseWriter, r *http.Request, config Config) {
scope, err := scopeFromQuery(r)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
return
}
view := libraryVerificationView{
Scope: scope,
History: []governance.LibraryRestoreVerification{},
Target: libraryVerificationTarget{
Instance: config.LibraryTargetInstance,
Project: config.LibraryTargetProject,
InternalKB: config.LibraryProductionInternalKB,
ExternalKB: config.LibraryProductionExternalKB,
Index: config.LibraryProductionIndex,
S3Bucket: config.LibraryProductionS3Bucket,
TenantID: config.LibraryProductionTenantID,
DatabaseSet: config.LibraryProductionDatabaseURL != "",
},
}
store, ok := config.Store.(interface {
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
LibraryRestoreVerifications(context.Context, string, int) ([]governance.LibraryRestoreVerification, error)
})
if !ok {
view.Reason = "当前部署没有恢复验证存储:无法读取证据"
writeJSON(w, http.StatusServiceUnavailable, view)
return
}
latest, found, err := store.LatestLibraryRestoreVerification(r.Context(), scope)
if err != nil {
view.Reason = "读取恢复验证证据失败:" + err.Error()
writeJSON(w, http.StatusServiceUnavailable, view)
return
}
if found {
view.Latest = &latest
view.JobID = latest.JobID
}
if history, err := store.LibraryRestoreVerifications(r.Context(), scope, 10); err == nil {
view.History = history
}
view.Verified, view.Reason = governance.LibraryVerificationSatisfied(latest, scope)
if !found {
view.Verified = false
view.Reason = "还没有对当前目标做过恢复验证:点「跑一次恢复验证」(真实备份 → canary → 恢复 → 复核)"
}
writeJSON(w, http.StatusOK, view)
}
// withRestoreEvidence 把"当前目标是否有真实恢复证据"注入 Config 的副本,
// 让同步/恢复闸口在同一个请求里按证据判断,而不是读一个可能被手填的 env。
// 读不到证据时不改写任何东西:闸口照旧因为缺证据而拒绝。
func (c Config) withRestoreEvidence(ctx context.Context, scope string) Config {
if ok, err := c.restoreEvidence(ctx, scope); err == nil {
c.restoreVerified = ok
c.restoreEvidenceReason = ""
} else {
c.restoreVerified = false
c.restoreEvidenceReason = err.Error()
}
return c
}
// restoreEvidence 读当前目标的最近一条证据并给出它能不能支撑"可恢复"。
func (c Config) restoreEvidence(ctx context.Context, scope string) (bool, error) {
store, ok := c.Store.(interface {
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
})
if !ok {
return false, errors.New("没有已核验的生产恢复演练证据")
}
latest, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
if err != nil {
return false, fmt.Errorf("读取恢复验证证据失败:%w", err)
}
if !found {
return false, errors.New("当前目标还没有恢复验证证据:先在「生产保护与恢复」里跑一次恢复验证")
}
ok, reason := governance.LibraryVerificationSatisfied(latest, scope)
if !ok {
return false, errors.New(reason)
}
return true, nil
}
// restoreGateReasons 是闸口对"恢复可验证"的统一判断:证据优先,
// 历史 env 覆盖仅在证据缺失时兜底(页面不再写入它)。
func restoreGateReasons(config Config) []string {
if config.restoreVerified || config.ProductionBackupRestoreVerified {
return nil
}
reason := config.restoreEvidenceReason
if reason == "" {
reason = "没有已核验的生产恢复演练证据"
}
return []string{reason}
}
+193
View File
@@ -0,0 +1,193 @@
package httpapi
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"knowledge-governance/internal/governance"
)
// 恢复验证的页面契约:
// - 只有管理员能发起(它会停写入者并在目标上写 canary);
// - 闸口关闭 / 维护中时直接拒绝,不投一个 worker 必然失败的作业;
// - 证据只由真实执行写入:没有证据 ⇒ verified=false,过期/换目标 ⇒ 失效。
//
// 这里不模拟"真跑一次恢复"(那需要真实目标);执行体本身在 internal/libraryverify,
// 由库内 CLI 与 worker 共用,属于隔离 fixture 之外的集成范围。
type verificationStore struct {
fakeLibraryStore
verifications []governance.LibraryRestoreVerification
}
func (s *verificationStore) LatestLibraryRestoreVerification(_ context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
for _, item := range s.verifications {
if item.Scope == scope {
return item, true, nil
}
}
return governance.LibraryRestoreVerification{}, false, nil
}
func (s *verificationStore) LibraryRestoreVerifications(_ context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
items := make([]governance.LibraryRestoreVerification, 0, len(s.verifications))
for _, item := range s.verifications {
if item.Scope == scope {
items = append(items, item)
}
}
if limit > 0 && len(items) > limit {
items = items[:limit]
}
return items, nil
}
func verifyRequest(body, role, username string) *http.Request {
request := httptest.NewRequest(http.MethodPost, "/api/v1/library/verifications", strings.NewReader(body))
request.Header.Set("Content-Type", "application/json")
ctx := context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: username, Role: role})
return request.WithContext(ctx)
}
func TestLibraryVerifyRequiresAdmin(t *testing.T) {
store := &verificationStore{}
recorder := httptest.NewRecorder()
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleEngineer, "eng"),
Config{Store: store, ProductionReleaseRequestsEnabled: true})
if recorder.Code != http.StatusForbidden {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
if store.createN != 0 {
t.Fatal("non-admin must not create a verify job")
}
}
func TestLibraryVerifyRefusesWhenWriteGateClosed(t *testing.T) {
store := &verificationStore{}
recorder := httptest.NewRecorder()
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
Config{Store: store, ProductionReleaseRequestsEnabled: false})
if recorder.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
body := decodeBody(t, recorder)
if body["code"] != "library_verify_blocked" {
t.Fatalf("body=%v", body)
}
if store.createN != 0 {
t.Fatal("closed gate must not create a verify job")
}
}
func TestLibraryVerifyEnqueuesVerifyJobWithCanaryStages(t *testing.T) {
store := &verificationStore{}
store.enqueueEchoRequestedID = true
recorder := httptest.NewRecorder()
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
Config{Store: store, ProductionReleaseRequestsEnabled: true})
if recorder.Code != http.StatusAccepted {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
if store.lastJob.Kind != governance.LibraryJobVerify || store.lastJob.Scope != "internal" {
t.Fatalf("job=%#v", store.lastJob)
}
if !strings.HasPrefix(store.lastJob.ID, "libverify-internal-") {
t.Fatalf("job id=%q", store.lastJob.ID)
}
// 阶段必须包含 canary:没有"写 canary 证明校验有效"这一步,验证就只是自证。
want := governance.VerifyStages()
if len(want) != 5 || want[2] != "canary" || want[3] != "restore" || want[4] != "finalize" {
t.Fatalf("verify stages=%v", want)
}
}
func TestLibraryVerificationsWithoutEvidenceIsNotVerified(t *testing.T) {
store := &verificationStore{}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
libraryVerifications(recorder, request, Config{Store: store})
if recorder.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
body := decodeBody(t, recorder)
if body["verified"] != false {
t.Fatalf("verified must be false without evidence: %v", body)
}
if reason, _ := body["reason"].(string); !strings.Contains(reason, "还没有对当前目标做过恢复验证") {
t.Fatalf("reason=%q", reason)
}
}
func TestLibraryVerificationsWithPassedEvidenceIsVerified(t *testing.T) {
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
ID: "libverify-internal-job1", Scope: "internal", Status: governance.LibraryVerificationPassed,
SnapshotID: "snap-1", JobID: "job1", StartedAt: time.Now().UTC().Add(-time.Minute),
FinishedAt: time.Now().UTC(),
}}}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
libraryVerifications(recorder, request, Config{Store: store})
body := decodeBody(t, recorder)
if body["verified"] != true {
t.Fatalf("verified must be true with passed evidence: %v", body)
}
}
func TestLibraryVerificationsFailedEvidenceIsNotVerified(t *testing.T) {
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
ID: "libverify-internal-job2", Scope: "internal", Status: governance.LibraryVerificationFailed,
SnapshotID: "snap-2", JobID: "job2", Failure: "恢复后内容仍与预备份不一致",
StartedAt: time.Now().UTC().Add(-time.Minute), FinishedAt: time.Now().UTC(),
}}}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
libraryVerifications(recorder, request, Config{Store: store})
body := decodeBody(t, recorder)
if body["verified"] != false {
t.Fatalf("failed evidence must not verify: %v", body)
}
if reason, _ := body["reason"].(string); !strings.Contains(reason, "恢复后内容仍与预备份不一致") {
t.Fatalf("reason=%q", reason)
}
}
// 同步闸口必须消费同一 target 的证据:有 passed 证据即放行,
// 只有失败证据时仍然拒绝并给出失败原因(不是笼统"没有证据")。
func TestLibrarySyncGateConsumesRestoreEvidence(t *testing.T) {
base := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40}
base.Store = &verificationStore{verifications: []governance.LibraryRestoreVerification{{
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1",
JobID: "job1", FinishedAt: time.Now().UTC(),
}}}
allowed := librarySyncGateWithGrant(base.withRestoreEvidence(context.Background(), "internal"),
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
if !allowed.SyncAllowed {
t.Fatalf("gate should allow with passed evidence: %v", allowed.Reasons)
}
noEvidence := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
Store: &verificationStore{}}.withRestoreEvidence(context.Background(), "internal")
blocked := librarySyncGateWithGrant(noEvidence,
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
if blocked.SyncAllowed {
t.Fatalf("gate without env and without evidence must block: %v", blocked.Reasons)
}
if len(blocked.Reasons) == 0 || !strings.Contains(strings.Join(blocked.Reasons, ";"), "恢复验证") {
t.Fatalf("blocked reasons=%v", blocked.Reasons)
}
failedConfig := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
Store: &verificationStore{verifications: []governance.LibraryRestoreVerification{{
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-2",
JobID: "job2", Failure: "canary 后校验仍通过", FinishedAt: time.Now().UTC(),
}}}}
reason := strings.Join(restoreGateReasons(failedConfig.withRestoreEvidence(context.Background(), "internal")), ";")
if !strings.Contains(reason, "canary 后校验仍通过") {
t.Fatalf("失败证据的失败原因必须出现在闸口:%q", reason)
}
}
+54
View File
@@ -6,6 +6,7 @@ import (
"strings"
"time"
"knowledge-governance/internal/pgdsn"
"knowledge-governance/internal/platform/weknora"
)
@@ -33,6 +34,55 @@ type productionBindingView struct {
ProductionBaseURL string `json:"production_base_url"`
ProductionConnected bool `json:"production_connected"`
ProductionMessage string `json:"production_message"`
// LegacyScope 说明这条记录属于哪条发布路径。整库发布(封存/同步/恢复)不读它,
// 只读下面的 WholeTarget;页面必须把两者分开写,否则会把旧文档级目标当成整库目标。
LegacyScope string `json:"legacy_scope"`
// WholeTarget 是整库发布真正写入的目标(来自部署环境 LIBRARY_PRODUCTION_*),
// 与上面可编辑的文档级绑定是两条独立路径。
WholeTarget libraryPublishTargetView `json:"whole_target"`
}
// libraryPublishTargetView 是"整库发布到底写哪儿"的只读投影。
// 为什么由后端给出:页面上的 kb 名称/文档数来自生产端列表接口,而整库写入用的是
// 另一组环境配置;让页面自己拼会把两条路径混成一个。
type libraryPublishTargetView struct {
Configured bool `json:"configured"`
Source string `json:"source"`
TenantID uint64 `json:"tenant_id"`
InternalKBID string `json:"internal_kb_id"`
ExternalKBID string `json:"external_kb_id"`
Database string `json:"database"`
ElasticIndex string `json:"elastic_index"`
ObjectBucket string `json:"object_bucket"`
MissingReason string `json:"missing_reason"`
}
// libraryPublishTarget 从 config 派生整库目标;缺项照实说缺哪一项,不猜默认值。
func (config Config) libraryPublishTarget() libraryPublishTargetView {
view := libraryPublishTargetView{
Source: "部署环境 LIBRARY_PRODUCTION_*",
TenantID: config.LibraryProductionTenantID,
InternalKBID: config.LibraryProductionInternalKB,
ExternalKBID: config.LibraryProductionExternalKB,
Database: pgdsn.Endpoint(config.LibraryProductionDatabaseURL),
ElasticIndex: config.LibraryProductionIndex,
ObjectBucket: config.LibraryProductionS3Bucket,
}
missing := make([]string, 0, 4)
if config.LibraryProductionDatabaseURL == "" {
missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL")
}
if view.InternalKBID == "" {
missing = append(missing, "LIBRARY_PRODUCTION_KB_INTERNAL")
}
if config.LibraryProductionIndex == "" {
missing = append(missing, "LIBRARY_PRODUCTION_INDEX")
}
view.Configured = len(missing) == 0
if len(missing) > 0 {
view.MissingReason = "未配置:" + strings.Join(missing, "、")
}
return view
}
// productionTargetBaseURL is the production endpoint the picker reads from: the
@@ -79,6 +129,10 @@ func (config Config) productionBindingPayload(ctx context.Context) (productionBi
InternalKBName: names[storedInternal], ExternalKBName: names[storedExternal],
UpdatedBy: updatedBy, ProductionBaseURL: config.productionTargetBaseURL(ctx),
ProductionConnected: message == "", ProductionMessage: message,
// 这张表服务的是"文档级发布"(单篇文档写生产)。整库发布读的是部署环境
// LIBRARY_PRODUCTION_*,两者不能互相顶替,所以页面必须分开显示。
LegacyScope: "文档级发布(单篇文档写生产)· 旧路径",
WholeTarget: config.libraryPublishTarget(),
}, nil
}
@@ -178,3 +178,50 @@ func TestProductionBindingOptionsReportUnavailableReason(t *testing.T) {
t.Fatalf("status = %d body %s", recorder.Code, recorder.Body.String())
}
}
// 页面必须说清"整库发布真正写哪个 KB/端点"。这条投影只能来自部署配置,
// 不能从可编辑的文档级绑定推出来,也不能把口令带进响应。
func TestLibraryPublishTargetIsDerivedFromDeploymentConfig(t *testing.T) {
config := Config{
// 只放形状,不放任何可用凭据:口令段是占位符,主机是 .invalid。
LibraryProductionDatabaseURL: "postgres://governance@db.invalid:5432/weknora_test?sslmode=disable",
LibraryProductionInternalKB: "a8e9c8ad-411c-4c53-8421-599d29e9b179",
LibraryProductionExternalKB: "2e980686-2233-4c4e-b5a6-8b0069477b8e",
LibraryProductionTenantID: 10000,
LibraryProductionIndex: "weknora_test_chunks",
LibraryProductionS3Bucket: "dobot-weknora-test",
}
target := config.libraryPublishTarget()
if !target.Configured || target.MissingReason != "" {
t.Fatalf("完整配置应判为已配置: %+v", target)
}
if target.InternalKBID != "a8e9c8ad-411c-4c53-8421-599d29e9b179" || target.TenantID != 10000 {
t.Fatalf("必须回显实际整库目标而不是占位: %+v", target)
}
if target.Database != "db.invalid:5432/weknora_test" {
t.Fatalf("端点应只含 host:port/db,得到 %q", target.Database)
}
// 回显只能是端点:连接串里的用户信息(口令、账号)一个字符都不能进响应。
if strings.Contains(target.Database, "@") || strings.Contains(target.Source, "@") ||
strings.Contains(target.Source, "postgres://") {
t.Fatalf("响应里不能出现连接串用户信息: %+v", target)
}
if target.ElasticIndex != "weknora_test_chunks" || target.ObjectBucket != "dobot-weknora-test" {
t.Fatalf("索引与桶也要显示: %+v", target)
}
}
func TestLibraryPublishTargetNamesMissingConfigInsteadOfGuessing(t *testing.T) {
target := Config{}.libraryPublishTarget()
if target.Configured {
t.Fatal("缺配置时不能报告已配置")
}
for _, key := range []string{"LIBRARY_PRODUCTION_DATABASE_URL", "LIBRARY_PRODUCTION_KB_INTERNAL"} {
if !strings.Contains(target.MissingReason, key) {
t.Fatalf("缺失说明应点名 %s,得到 %q", key, target.MissingReason)
}
}
if target.InternalKBID != "" || target.Database != "(未配置)" {
t.Fatalf("缺配置就不要编造目标: %+v", target)
}
}
+18 -2
View File
@@ -181,6 +181,9 @@ type Config struct {
LibraryProductionInternalKB string
LibraryProductionExternalKB string
LibraryProductionTenantID uint64
// 整库发布还落在这些后端上;页面要显示"到底写哪儿",只给 KB ID 不够。
LibraryProductionIndex string
LibraryProductionS3Bucket string
// LibraryScopeProbe 让测试注入"目标整库是否为空"的探测;留空即用真实连接。
// 它不是运行时开关:生产部署不设置它,那次探测仍然直连目标库逐表计数。
LibraryScopeProbe func(ctx context.Context, scope string) (bool, map[string]int64, error)
@@ -198,8 +201,13 @@ type Config struct {
RequireWeKnoraReady bool
ProductionReleaseRequestsEnabled bool
// TestAutoReviewPolicy 只用于展示:让运维在界面上看到"现在自动批准到什么程度"。
TestAutoReviewPolicy string
ProductionBackupRestoreVerified bool
TestAutoReviewPolicy string
ProductionBackupRestoreVerified bool
// restoreVerified / restoreEvidenceReason 由请求期的证据读取填入(withRestoreEvidence):
// 闸口按"当前目标是否有真实恢复验证证据"判断,而不是按可手填的 env。
// 它们不可由配置注入,避免把证据变成另一个开关。
restoreVerified bool
restoreEvidenceReason string
ProductionReleaseAuthorizerRef string
ProductionReleaseAuthorizationToken string
// DingtalkSources 枚举「我账号下能看到」的钉钉知识库与钉盘空间(只读,dws)。
@@ -556,6 +564,14 @@ func NewHandler(config Config) (http.Handler, error) {
mux.HandleFunc("POST /api/v1/library/baselines", func(w http.ResponseWriter, r *http.Request) {
libraryBaselineEnqueue(w, r, config)
})
// 恢复验证:真实备份 → canary → 恢复 → 复核,通过后证据按当前目标登记,
// 同步闸口据此放行(不再需要页面/环境变量手填 verified)。
mux.HandleFunc("POST /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
libraryVerifyEnqueue(w, r, config)
})
mux.HandleFunc("GET /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
libraryVerifications(w, r, config)
})
// 全部 main 提交时间线(读本库缓存,不直接打 Gitea)
mux.HandleFunc("GET /api/v1/version-history", func(w http.ResponseWriter, r *http.Request) {
versionHistoryList(w, r, config)