diff --git a/CHANGELOG.md b/CHANGELOG.md
index fc36ce4..eb4a075 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,20 @@
# Changelog
+## 开发快照(B1-R1 / B2-R1 / B5-R1 真实链收口)
+
+### Fixed
+
+- **B1-R1 受限演练恢复计划**:写集合新增 `restore-drill` 种类——共享 V2 信封、contract 1 与 schema 摘要,
+ 对象/索引/本地 ID 映射必须为空,来源 tenant/KB 必填;任一项非空即拒绝,旧执行器不能把它当发布计划
+ 接管。恢复演练入口 `RestoreDrill` 与对象网关签发的零对象计划使用同一 attempt/写前镜像身份。
+- **B2-R1 verify 作业的不可变 operation**:`POST /api/v1/library/verifications` 在创建作业前先建
+ operation(`ensureLibraryIntent`),作业携带 `operation_id` 与稳定幂等键;同一幂等键对应不同请求返回
+ 409。worker owned attempt 与同 job recover 的网关 claim 都按原 operation + recovery job/operation
+ 核验,缺 operation 的 verify 作业在真实恢复中 fail closed。
+- **B5-R1 迟到响应不得污染新 scope**:恢复验证面板按 scope 世代 + 动作序号判定在飞动作;切 scope 即作废
+ 旧 POST/回调,迟到响应不能改写新 scope 观测、不能触发父级旧 job 回调、不能清掉新动作的 pending。
+ 新增真实挂载反例用例覆盖旧 internal POST 晚到的场景。
+
## 开发快照(GPT-6 正常 UI 发布审核 B1–B5 修复)
### Fixed
diff --git a/cmd/worker/library_three_blockers_chain_test.go b/cmd/worker/library_three_blockers_chain_test.go
new file mode 100644
index 0000000..e0e22e0
--- /dev/null
+++ b/cmd/worker/library_three_blockers_chain_test.go
@@ -0,0 +1,1202 @@
+package main
+
+// 开发隔离集成:B1-R1 / B2-R1 的真实纵向链。
+//
+// 只由外置 runner 在一次性资源上启动(KG_THREE_RUN 非空)。它使用真实
+// PostgreSQL / Elasticsearch / MinIO、真实对象网关与入口容器,走正式共享装配
+// (librarystack.OpenBaselineStack / OpenProductionStack,Automatic 网关保留)。
+// 链路:
+// API POST /library/verifications
+// -> 治理 operation/job(B2)
+// -> 真实 owned attempt(worker claim)
+// -> 两次真实 Prepare + 受限演练计划/ledger 写前登记
+// -> canary 漂移
+// -> 四阶段真实恢复(database/wiki/resources/index)
+// -> 内容核验 + 真实入口恢复/维护释放(B1)
+// 失败后:API POST /library/maintenance/recover -> 真实 owned attempt
+// -> 同一 job/backup/原 operation + 非 nil DS01 gate -> RestoreDrill(B2)
+// -> 网关按 recovery claim 校验零对象计划。
+//
+// 这不是 8088 页面验收,也不是生产执行;所有资源由 runner 创建前登记、清理按
+// 不可变 ID/ownership/readback。
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/cookiejar"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "golang.org/x/crypto/bcrypt"
+
+ "knowledge-governance/internal/appconfig"
+ "knowledge-governance/internal/governance"
+ "knowledge-governance/internal/jobs"
+ "knowledge-governance/internal/libraryrelease"
+ "knowledge-governance/internal/objectgateway"
+ postgresstore "knowledge-governance/internal/platform/postgres"
+ v071 "knowledge-governance/internal/releaseadapter/v071"
+)
+
+const tbTargetTenant = 31001
+const tbTargetKB = "kb-verify-chain"
+const tbSentinelTenant = 31999
+const tbSentinelKB = "kb-sentinel"
+
+const tbSchema = `
+CREATE TABLE schema_migrations(version int, dirty bool);
+INSERT INTO schema_migrations VALUES(74,false);
+CREATE TABLE knowledge_bases(id text PRIMARY KEY, tenant_id bigint, indexing_strategy jsonb, deleted_at timestamptz,
+ embedding_model_id text, vector_store_id text);
+CREATE TABLE models(id text PRIMARY KEY, name text, parameters jsonb, deleted_at timestamptz);
+CREATE TABLE vector_stores(id text PRIMARY KEY, engine_type text, index_config jsonb, deleted_at timestamptz);
+CREATE TABLE knowledges(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, parse_status text,
+ pending_subtasks_count int, file_path text, deleted_at timestamptz);
+CREATE TABLE chunks(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, knowledge_id text,
+ content text, seq_id bigint, deleted_at timestamptz);
+CREATE TABLE knowledge_tags(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, name text, deleted_at timestamptz);
+CREATE TABLE knowledge_tag_relations(knowledge_id text, tag_id text, tenant_id bigint);
+CREATE TABLE resources(id text PRIMARY KEY, handle text, tenant_id bigint, physical_path text, provider text,
+ size bigint, storage_backend_id text, deleted_at timestamptz);
+CREATE TABLE resource_bindings(id text PRIMARY KEY, tenant_id bigint, resource_id text, owner_type text,
+ owner_id text, relation text);
+CREATE TABLE wiki_folders(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, deleted_at timestamptz);
+CREATE TABLE wiki_pages(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, slug text, content text,
+ status text, deleted_at timestamptz);
+CREATE TABLE wiki_page_issues(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, deleted_at timestamptz);
+CREATE TABLE wiki_log_entries(id bigint PRIMARY KEY, tenant_id bigint, knowledge_base_id text);
+CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz);
+CREATE TABLE task_pending_ops(tenant_id bigint, task_type text, scope text, scope_id text);
+`
+
+type tbEnv struct {
+ root string
+ pg *pgxpool.Pool
+ gov *pgxpool.Pool
+ store *postgresstore.Store
+ config appconfig.Config
+ api string
+ ingress string
+ gatewayAddr string
+ bin string
+ project string
+ index string
+ bucket string
+ s3Endpoint string
+ s3Access string
+ s3Secret string
+ readerKey string
+ readerSec string
+ archiveRoot string
+ workerID string
+}
+
+func tbGetenv(t *testing.T, name string) string {
+ t.Helper()
+ value := os.Getenv(name)
+ if value == "" {
+ t.Fatalf("missing required isolated env %s", name)
+ }
+ return value
+}
+
+func tbWriteJSON(t *testing.T, root, name string, value any) {
+ t.Helper()
+ raw, err := json.MarshalIndent(value, "", " ")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(root, name), append(raw, '\n'), 0600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func tbStart(t *testing.T, root, label, binary string, env map[string]string) *exec.Cmd {
+ t.Helper()
+ cmd := exec.Command(binary)
+ cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")}
+ for key, value := range env {
+ cmd.Env = append(cmd.Env, key+"="+value)
+ }
+ log, err := os.OpenFile(filepath.Join(root, label+".log"), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cmd.Stdout, cmd.Stderr = log, log
+ if err := cmd.Start(); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+ log.Close()
+ })
+ return cmd
+}
+
+func tbS3Locator(tenant uint64, id string) string {
+ return fmt.Sprintf("storage://backend-target/minio://%s/%d/%s/a.png", tbGetenvID("KG_THREE_BUCKET"), tenant, id)
+}
+
+// tbWaitHTTP 等待前台服务真正绑定端口后再发业务请求;失败时把服务日志带进断言。
+func tbWaitHTTP(t *testing.T, url, logPath string, timeout time.Duration) {
+ t.Helper()
+ deadline := time.Now().Add(timeout)
+ for {
+ resp, err := http.Get(url)
+ if err == nil {
+ resp.Body.Close()
+ if resp.StatusCode >= 200 && resp.StatusCode < 500 {
+ return
+ }
+ }
+ if time.Now().After(deadline) {
+ raw, _ := os.ReadFile(logPath)
+ t.Fatalf("service never became ready: %s (%v); log: %s", url, err, strings.TrimSpace(string(raw)))
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+}
+
+func tbGetenvID(name string) string {
+ if v := os.Getenv(name); v != "" {
+ return v
+ }
+ return "fixture-bucket"
+}
+
+func TestThreeBlockersVerifyChain(t *testing.T) {
+ root := os.Getenv("KG_THREE_RUN")
+ if root == "" {
+ t.Skip("requires the dedicated isolated three-blockers runner (KG_THREE_RUN)")
+ }
+ if os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
+ t.Fatal("isolated three-blockers chain requires KG_LIBRARY_TEST_ISOLATED=1")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute)
+ t.Cleanup(cancel)
+
+ env := &tbEnv{
+ root: root, api: tbGetenv(t, "KG_THREE_API_ADDR"), ingress: tbGetenv(t, "KG_THREE_INGRESS_URL"),
+ gatewayAddr: tbGetenv(t, "KG_THREE_GATEWAY_ADDR"), bin: tbGetenv(t, "KG_THREE_BIN"),
+ project: tbGetenv(t, "KG_THREE_PROJECT"), index: tbGetenv(t, "KG_THREE_INDEX"),
+ bucket: tbGetenv(t, "KG_THREE_BUCKET"), s3Endpoint: tbGetenv(t, "KG_LIBRARY_TEST_S3_ENDPOINT"),
+ s3Access: tbGetenv(t, "KG_LIBRARY_TEST_S3_ACCESS_KEY"), s3Secret: tbGetenv(t, "KG_LIBRARY_TEST_S3_SECRET_KEY"),
+ readerKey: tbGetenv(t, "KG_THREE_READER_KEY"), readerSec: tbGetenv(t, "KG_THREE_READER_SECRET"),
+ archiveRoot: filepath.Join(root, "archive"), workerID: tbGetenv(t, "KG_THREE_WORKER_ID"),
+ }
+ if err := os.MkdirAll(env.archiveRoot, 0700); err != nil {
+ t.Fatal(err)
+ }
+
+ // ---------- 1) 业务库 / 目标现场 ----------
+ pool, err := pgxpool.New(ctx, tbGetenv(t, "KG_LIBRARY_TEST_DATABASE_URL"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(pool.Close)
+ env.pg = pool
+ if _, err := pool.Exec(ctx, tbSchema); err != nil {
+ t.Fatalf("business schema: %v", err)
+ }
+ tbSeedTarget(t, ctx, env)
+
+ // ---------- 2) 治理库 / 网关 / API ----------
+ gov, err := pgxpool.New(ctx, tbGetenv(t, "KG_THREE_GOVERNANCE_DSN"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(gov.Close)
+ env.gov = gov
+ if err := postgresstore.ApplyMigrations(ctx, gov); err != nil {
+ t.Fatalf("governance migrations: %v", err)
+ }
+ if _, err := gov.Exec(ctx, objectgateway.Schema); err != nil {
+ t.Fatalf("gateway schema: %v", err)
+ }
+ // 业务/API/worker/网关必须共享同一个 target 身份:API 建库作业时写的是
+ // appconfig.LibraryTargetIdentity(),worker 的 LibraryJob 查询按它过滤。
+ workerEnv := tbConfigEnv(env)
+ config, err := appconfig.Load(func(key string) string { return workerEnv[key] })
+ if err != nil {
+ t.Fatalf("app config: %v", err)
+ }
+ env.config = config
+ targetID := config.LibraryTargetIdentity()
+ env.store = postgresstore.NewStore(gov).WithLibraryTarget(targetID)
+ readerDSN, readerRole := tbCreateReaderRole(t, ctx, gov, tbGetenv(t, "KG_THREE_GOVERNANCE_DSN"))
+ tbProbeReaderPrivileges(t, ctx, root, readerDSN, readerRole)
+ tbStartGateway(t, ctx, env, readerDSN, targetID)
+
+ // 入口/写者容器由 runner 建好;先证明拓扑与原状态可读。
+ manager, err := openLibraryMaintenance(config)
+ if err != nil {
+ t.Fatalf("maintenance wiring: %v", err)
+ }
+ if err := manager.VerifyTopology(ctx); err != nil {
+ t.Fatalf("topology: %v", err)
+ }
+ manager.Close()
+
+ apiEnv := map[string]string{}
+ for key, value := range workerEnv {
+ apiEnv[key] = value
+ }
+ apiEnv["APP_ADDR"] = env.api
+ apiEnv["GOVERNANCE_SPOOL_ROOT"] = filepath.Join(root, "spool")
+ apiEnv["RELEASE_WORKER_ENABLED"] = "false"
+ if err := os.MkdirAll(apiEnv["GOVERNANCE_SPOOL_ROOT"], 0700); err != nil {
+ t.Fatal(err)
+ }
+ tbStart(t, root, "api", filepath.Join(env.bin, "api"), apiEnv)
+ tbWaitHTTP(t, "http://"+env.api+"/health", filepath.Join(root, "api.log"), 60*time.Second)
+ api := tbLogin(t, ctx, gov, "http://"+env.api)
+
+ // ---------- 3) 正常完整验证链路 ----------
+ beforePG := tbDump(t, ctx, pool)
+ beforeIndex := tbIndexReadback(t, ctx, env)
+ tbWriteJSON(t, root, "10-before-pg.json", beforePG)
+ tbWriteJSON(t, root, "11-before-index.json", beforeIndex)
+
+ job1 := tbEnqueueVerify(t, api)
+ tbWriteJSON(t, root, "12-verify-job.json", job1)
+ if job1.OperationID == "" {
+ t.Fatal("API-created verify job carries no operation_id (B2-R1)")
+ }
+ queue1 := tbClaimQueue(t, ctx, env.store, "library.verify")
+ if err := runLibraryVerify(ctx, env.store, config, *queue1); err != nil {
+ t.Fatalf("normal verify failed: %v", err)
+ }
+ after1, err := env.store.LibraryJob(ctx, job1.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ tbWriteJSON(t, root, "13-verify-job-after.json", after1)
+ if after1.State != governance.LibraryJobSucceeded {
+ t.Fatalf("normal verify job state=%s error=%s", after1.State, after1.Error)
+ }
+ evidence, found, err := env.store.LatestLibraryRestoreVerification(ctx, "internal")
+ if err != nil || !found {
+ t.Fatalf("verification evidence missing: found=%v err=%v", found, err)
+ }
+ tbWriteJSON(t, root, "14-verification-evidence.json", evidence)
+ if evidence.Status != governance.LibraryVerificationPassed || evidence.JobID != job1.ID {
+ t.Fatalf("verification evidence=%#v", evidence)
+ }
+ // 内容与 sentinel:恢复后必须逐行/逐文档回到写前镜像,且无关租户不变。
+ if diff := tbDiffDump(beforePG, tbDump(t, ctx, pool)); diff != "" {
+ t.Fatalf("restored business content differs: %s", diff)
+ }
+ if diff := tbDiffIndex(beforeIndex, tbIndexReadback(t, ctx, env)); diff != "" {
+ t.Fatalf("restored index differs: %s", diff)
+ }
+ tbAssertMaintenanceReleased(t, ctx, env)
+ tbAssertIngressHealthy(t, env)
+ // 演练计划:受限 V2、零对象/索引/本地映射,且已登记为 production_prebackup。
+ ledger := tbFindDrillLedger(t, ctx, env, job1.ID)
+ tbWriteJSON(t, root, "15-drill-plan.json", map[string]any{
+ "digest": ledger.Digest, "job": ledger.JobID, "kind": ledger.Kind, "version": ledger.Version,
+ "contract_version": ledger.ContractVersion, "objects": len(ledger.Objects), "index": len(ledger.Index),
+ "local_ids": len(ledger.LocalIDs), "backup_snapshot": ledger.BackupSnapshotID,
+ "target_snapshot": ledger.TargetSnapshotID, "rows": ledger.Rows,
+ "schema_digest": ledger.SchemaDigest,
+ })
+ if ledger.Kind != libraryrelease.WriteSetKindRestoreDrill || ledger.Version != libraryrelease.MigrationWriteSetVersion ||
+ len(ledger.Objects) != 0 || len(ledger.Index) != 0 || len(ledger.LocalIDs) != 0 {
+ t.Fatalf("drill plan is not restricted: %#v", ledger)
+ }
+ // 对象网关确实签发了零对象计划(controller delivery 行),不是跳过网关。
+ var deliveries int
+ if err := gov.QueryRow(ctx, `SELECT count(*) FROM kg_object_gateway_deliveries WHERE job=$1`, job1.ID).Scan(&deliveries); err != nil {
+ t.Fatal(err)
+ }
+ if deliveries != 1 {
+ t.Fatalf("object gateway deliveries for verify job = %d, want 1", deliveries)
+ }
+
+ // ---------- 4) 失败后同一 job 恢复链路(DS01 gate 非 nil) ----------
+ proxy := tbNewRefusingProxy(t, env, tbGetenv(t, "KG_LIBRARY_TEST_ELASTIC_URL"))
+ // 让 worker 装配指向代理:其余字段不变。
+ workerEnv["LIBRARY_PRODUCTION_ELASTIC_URL"] = proxy.url
+ config2, err := appconfig.Load(func(key string) string { return workerEnv[key] })
+ if err != nil {
+ t.Fatal(err)
+ }
+ config = config2
+
+ job2 := tbEnqueueVerify(t, api)
+ tbWriteJSON(t, root, "20-recover-job.json", job2)
+ queue2 := tbClaimQueue(t, ctx, env.store, "library.verify")
+ done := make(chan error, 1)
+ go func() { done <- runLibraryVerify(ctx, env.store, config, *queue2) }()
+ tbWaitStage(t, ctx, env, job2.ID, "restore", 10*time.Minute)
+ proxy.RefuseNextBulk()
+ runErr := <-done
+ if runErr == nil {
+ t.Fatal("verify with injected ES refusal after canary unexpectedly succeeded")
+ }
+ after2, err := env.store.LibraryJob(ctx, job2.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ tbWriteJSON(t, root, "21-recover-job-failed.json", after2)
+ tbWriteJSON(t, root, "22-recover-run-error.json", map[string]string{"error": runErr.Error()})
+ if after2.State != governance.LibraryJobFailed {
+ t.Fatalf("failed verify state=%s", after2.State)
+ }
+ inst, err := env.store.InstanceMaintenanceOf(ctx, env.project)
+ if err != nil || inst == nil || inst.ReleasedAt != nil || inst.State != "needs_restore" || inst.IngressState != "closed" {
+ t.Fatalf("held instance maintenance = %#v err=%v", inst, err)
+ }
+ if err := tbIngressClosed(env); err != nil {
+ t.Fatalf("ingress must be closed during held window: %v", err)
+ }
+ // 真实 UI/API 恢复入口:同一个原窗口 job。
+ recoveryQueue := tbPostRecover(t, api, job2.ID)
+ tbWriteJSON(t, root, "23-recover-queue.json", recoveryQueue)
+ proxy.Disarm()
+ // 真实消费路径:与生产 worker 同一个 jobs.Worker 外壳
+ // (claim -> handler -> heartbeat -> Complete/Fail)。
+ // 直接调用 handler 只会执行恢复函数,不会写回 jobs 行的正式终态,页面/队列
+ // 会永远看到 running;这里必须让同一外壳完成并按 id 读回终态。
+ recoveryShell, err := jobs.NewWorker(jobs.WorkerConfig{
+ Queue: env.store,
+ WorkerID: config.WorkerID,
+ LeaseDuration: config.LeaseDuration,
+ RetryDelay: config.RetryDelay,
+ Handlers: map[string]jobs.Handler{
+ libraryRecoverJobType: admittanceGuarded(nil, func(ctx context.Context, job jobs.Job) error {
+ return runLibraryRecovery(ctx, env.store, config, job)
+ }),
+ },
+ })
+ if err != nil {
+ t.Fatalf("recovery worker shell: %v", err)
+ }
+ claimed, err := recoveryShell.RunOnce(ctx)
+ if err != nil {
+ t.Fatalf("same-job recovery through queue shell failed: %v", err)
+ }
+ if !claimed {
+ t.Fatalf("recovery queue %s was not claimed by the worker shell", recoveryQueue.ID)
+ }
+ queueFinal := tbWaitQueueTerminal(t, ctx, env.store, recoveryQueue.ID)
+ tbWriteJSON(t, root, "24-recover-complete.json", map[string]any{"window_job": job2.ID, "recovery_queue": recoveryQueue.ID, "terminal": queueFinal})
+ if queueFinal.State != jobs.StateCompleted || queueFinal.LeaseOwner != "" || queueFinal.LastError != "" {
+ t.Fatalf("recovery queue terminal state=%s owner=%q last_error=%q, want completed with released lease",
+ queueFinal.State, queueFinal.LeaseOwner, queueFinal.LastError)
+ }
+ if diff := tbDiffDump(beforePG, tbDump(t, ctx, pool)); diff != "" {
+ t.Fatalf("recovery business content differs: %s", diff)
+ }
+ if diff := tbDiffIndex(beforeIndex, tbIndexReadback(t, ctx, env)); diff != "" {
+ t.Fatalf("recovery index differs: %s", diff)
+ }
+ tbAssertMaintenanceReleased(t, ctx, env)
+ tbAssertIngressHealthy(t, env)
+ // 网关 delivery 必须记录 recovery 身份(原 attempt + recovery job/operation),
+ // 证明不是把 gate 置 nil 绕过。
+ var claimRaw []byte
+ if err := gov.QueryRow(ctx, `SELECT claim FROM kg_object_gateway_deliveries WHERE job=$1 ORDER BY attempt_epoch DESC LIMIT 1`, job2.ID).Scan(&claimRaw); err != nil {
+ t.Fatal(err)
+ }
+ var claim objectgateway.Claim
+ if err := json.Unmarshal(claimRaw, &claim); err != nil {
+ t.Fatal(err)
+ }
+ tbWriteJSON(t, root, "25-recovery-claim.json", claim)
+ if claim.OperationID != job2.OperationID || claim.RecoveryJobID == "" || claim.RecoveryOperationID == "" {
+ t.Fatalf("recovery gateway claim does not carry original+recovery identity: %#v", claim)
+ }
+ // 终态读回:原失败作业保持 failed(历史语义不改写,不要求它变成功);
+ // recovery 队列终态由同一消费外壳写回,且已失租。
+ finalWindow, err := env.store.LibraryJob(ctx, job2.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if finalWindow.State != governance.LibraryJobFailed {
+ t.Fatalf("original failed window job was rewritten to %s (must stay failed)", finalWindow.State)
+ }
+ finalInst, err := env.store.InstanceMaintenanceOf(ctx, env.project)
+ if err != nil || finalInst == nil || finalInst.ReleasedAt == nil ||
+ finalInst.State != "released" || finalInst.IngressState != "open" {
+ t.Fatalf("instance maintenance final = %#v err=%v", finalInst, err)
+ }
+ var scopeState, scopeIngress string
+ var scopeReleased *time.Time
+ if err := gov.QueryRow(ctx, `SELECT state,ingress_state,released_at FROM library_maintenance WHERE scope='internal'`).
+ Scan(&scopeState, &scopeIngress, &scopeReleased); err != nil {
+ t.Fatal(err)
+ }
+ if scopeReleased == nil || scopeState != "verified" || scopeIngress != "open" {
+ t.Fatalf("scope maintenance final state=%s ingress=%s released=%v", scopeState, scopeIngress, scopeReleased)
+ }
+ sentinelPG := tbUnrelatedSentinel(t, ctx, pool)
+ if sentinelPG != "sentinel-doc/sentinel-chunk" {
+ t.Fatalf("unrelated PG sentinel changed: %s", sentinelPG)
+ }
+ sentinelIndex := "missing"
+ if _, ok := tbIndexReadback(t, ctx, env)["sentinel-chunk"]; ok {
+ sentinelIndex = "sentinel-chunk"
+ }
+ if sentinelIndex != "sentinel-chunk" {
+ t.Fatal("unrelated index sentinel missing after recovery")
+ }
+ tbWriteJSON(t, root, "26-final-state.json", map[string]any{
+ "window_job": job2.ID, "window_state": string(finalWindow.State),
+ "recovery_queue": recoveryQueue.ID, "recovery_queue_state": string(queueFinal.State),
+ "recovery_queue_lease_owner": queueFinal.LeaseOwner,
+ "consumer": "jobs.Worker.RunOnce (claim -> handler -> heartbeat -> Complete/Fail)",
+ "maintenance": map[string]any{
+ "instance_state": finalInst.State, "instance_ingress": finalInst.IngressState,
+ "instance_released_at": finalInst.ReleasedAt,
+ "scope_state": scopeState, "scope_ingress": scopeIngress, "scope_released_at": scopeReleased,
+ },
+ "ingress_healthy": true, "content_identical": true, "index_identical": true,
+ "unrelated_sentinel": map[string]string{"pg": sentinelPG, "index": sentinelIndex},
+ "claim": claim,
+ })
+}
+
+// tbSeedTarget 造一个有内容的隔离目标:文档/分块/资源/绑定/Wiki + 一个无关租户 sentinel。
+func tbSeedTarget(t *testing.T, ctx context.Context, env *tbEnv) {
+ t.Helper()
+ locator := func(id string) string {
+ return fmt.Sprintf("storage://backend-target/minio://%s/%d/%s/a.png", env.bucket, tbTargetTenant, id)
+ }
+ // 对象字节由 runner 用 admin 凭据写入(worker 只有只读身份)。
+ statements := []string{
+ `INSERT INTO knowledge_bases VALUES('` + tbTargetKB + `',31001,'{}',NULL,'fixture-model','fixture-vector')`,
+ `INSERT INTO models VALUES('fixture-model','fixture-embed','{"embedding_parameters":{"dimension":1024}}',NULL)`,
+ `INSERT INTO vector_stores VALUES('fixture-vector','elasticsearch_v8','{}',NULL)`,
+ `INSERT INTO storage_backends VALUES('backend-target',31001,'minio','{"bucket_name":"` + env.bucket + `"}',now())`,
+ `INSERT INTO knowledges VALUES('doc-target',31001,'` + tbTargetKB + `','completed',0,'',NULL)`,
+ `INSERT INTO chunks VALUES('chunk-target',31001,'` + tbTargetKB + `','doc-target','图:resource://aaaaaaaaaaaaaaaaaaaaaa',10,NULL)`,
+ `INSERT INTO knowledge_tags VALUES('tag-target',31001,'` + tbTargetKB + `','目标标签',NULL)`,
+ `INSERT INTO resources VALUES('res-target','aaaaaaaaaaaaaaaaaaaaaa',31001,'` + locator("res-target") + `','minio',8,'backend-target',NULL)`,
+ `INSERT INTO resource_bindings VALUES('bind-target',31001,'res-target','knowledge','doc-target','attachment')`,
+ `INSERT INTO wiki_pages(id,tenant_id,knowledge_base_id,slug,content,status) VALUES('wiki-target',31001,'` + tbTargetKB + `','index','目标 wiki','published')`,
+ `INSERT INTO wiki_folders VALUES('folder-target',31001,'` + tbTargetKB + `',NULL)`,
+ `INSERT INTO wiki_page_issues VALUES('issue-target',31001,'` + tbTargetKB + `',NULL)`,
+ `INSERT INTO wiki_log_entries VALUES(1,31001,'` + tbTargetKB + `')`,
+ // 无关 sentinel:另一个租户/知识库,恢复不允许碰。
+ `INSERT INTO knowledges VALUES('sentinel-doc',31999,'` + tbSentinelKB + `','completed',0,'',NULL)`,
+ `INSERT INTO chunks VALUES('sentinel-chunk',31999,'` + tbSentinelKB + `','sentinel-doc','sentinel 正文',1,NULL)`,
+ }
+ for _, statement := range statements {
+ if _, err := env.pg.Exec(ctx, statement); err != nil {
+ t.Fatalf("seed target: %v\n%s", err, statement)
+ }
+ }
+ // 索引:目标文档 + sentinel 文档;runner 已创建映射。
+ tbIndexDoc(t, ctx, env, "chunk-target", tbTargetTenant, tbTargetKB, "doc-target", "图:aaaaaaaaaaaaaaaaaaaaaa")
+ tbIndexDoc(t, ctx, env, "sentinel-chunk", tbSentinelTenant, tbSentinelKB, "sentinel-doc", "sentinel 正文")
+ // 对象:目标资源 + sentinel 对象。
+ tbPutObject(t, ctx, env, locator("res-target"), []byte("12345678"))
+}
+
+func tbIndexDoc(t *testing.T, ctx context.Context, env *tbEnv, id string, tenant uint64, kb, knowledge, content string) {
+ t.Helper()
+ vector := strings.TrimSuffix(strings.Repeat("0.1,", 1024), ",")
+ body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":[%s]}`,
+ tenant, kb, knowledge, id, id, content, vector)
+ req, err := http.NewRequestWithContext(ctx, http.MethodPut,
+ fmt.Sprintf("%s/%s/_doc/%s?refresh=true", tbGetenv(t, "KG_LIBRARY_TEST_ELASTIC_URL"), env.index, id), strings.NewReader(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("index doc: %v", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 && resp.StatusCode != 201 {
+ raw, _ := io.ReadAll(resp.Body)
+ t.Fatalf("index doc %s => %d %s", id, resp.StatusCode, raw)
+ }
+}
+
+func tbPutObject(t *testing.T, ctx context.Context, env *tbEnv, locator string, body []byte) {
+ t.Helper()
+ store, err := v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{Provider: "minio", Endpoint: env.s3Endpoint,
+ Region: "us-east-1", AccessKeyID: env.s3Access, SecretAccessKey: env.s3Secret, Bucket: env.bucket,
+ ForcePathStyle: true, MaxObjectBytes: 64 << 20})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := store.EnsureBucket(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.Put(ctx, "minio", locator, bytes.NewReader(body), int64(len(body))); err != nil {
+ t.Fatalf("put object %s: %v", locator, err)
+ }
+}
+
+func tbConfigEnv(env *tbEnv) map[string]string {
+ return map[string]string{
+ "KG_LIBRARY_TEST_ISOLATED": "1", "DATABASE_URL": tbGetenvID("KG_THREE_GOVERNANCE_DSN"),
+ "WORKER_ID": env.workerID, "LIBRARY_WORKER_ROLE": "release-executor",
+ "WORKER_POLL_INTERVAL": "200ms", "WORKER_LEASE_DURATION": "60s",
+ "PRODUCTION_RELEASE_REQUESTS_ENABLED": "true",
+ "PRODUCTION_RELEASE_AUTHORIZER_REF": env.project,
+ "PRODUCTION_RELEASE_AUTH_TOKEN_FILE": filepath.Join(env.root, "authorization-token"),
+ "LIBRARY_ARCHIVE_ROOT": env.archiveRoot,
+ "LIBRARY_CAPACITY_BUDGET_BYTES": "1073741824",
+ "LIBRARY_PRODUCTION_DATABASE_URL": tbGetenvID("KG_LIBRARY_TEST_DATABASE_URL"),
+ "LIBRARY_PRODUCTION_ELASTIC_URL": tbGetenvID("KG_LIBRARY_TEST_ELASTIC_URL"),
+ "LIBRARY_PRODUCTION_INDEX": env.index,
+ "LIBRARY_PRODUCTION_TENANT_ID": fmt.Sprint(tbTargetTenant),
+ "LIBRARY_PRODUCTION_KB_INTERNAL": tbTargetKB,
+ "LIBRARY_PRODUCTION_S3_PROVIDER": "minio",
+ "LIBRARY_PRODUCTION_S3_ENDPOINT": env.s3Endpoint,
+ "LIBRARY_PRODUCTION_S3_BUCKET": env.bucket,
+ "LIBRARY_PRODUCTION_S3_ACCESS_KEY": env.readerKey,
+ "LIBRARY_PRODUCTION_S3_SECRET_KEY": env.readerSec,
+ "LIBRARY_TARGET_INSTANCE": env.project,
+ "LIBRARY_TARGET_PROJECT": env.project,
+ "LIBRARY_TARGET_WRITERS": "writer",
+ "LIBRARY_TARGET_NON_WRITERS": "ingress=entry gate,redis=queue,minio=object backend",
+ "LIBRARY_TARGET_INGRESS_URL": env.ingress,
+ "LIBRARY_TARGET_INGRESS_SERVICE": "ingress",
+ "LIBRARY_TARGET_INGRESS_DRAIN_FILE": filepath.Join(env.root, "gate/drain.json"),
+ "LIBRARY_TARGET_DOCKER_SOCKET": "/var/run/docker.sock",
+ "LIBRARY_TARGET_REDIS_ADDR": tbGetenvID("KG_THREE_REDIS_ADDR"),
+ "LIBRARY_TARGET_REDIS_PASSWORD": tbGetenvID("KG_THREE_REDIS_PASSWORD"),
+ "LIBRARY_TARGET_REDIS_DB": "0",
+ "LIBRARY_TARGET_JOURNAL_ROOT": filepath.Join(env.root, "maintenance"),
+ "LIBRARY_EMBEDDING_MODEL": "fixture-model",
+ "LIBRARY_EMBEDDING_DIMENSION": "1024",
+ "LIBRARY_RESTORE_DISPATCH_V2": "1",
+ "LIBRARY_OBJECT_GATEWAY_URL": "http://" + env.gatewayAddr,
+ "LIBRARY_OBJECT_GATEWAY_TOKEN": tbGetenvID("KG_THREE_DATA_TOKEN"),
+ "LIBRARY_OBJECT_GATEWAY_GRANT_DIRECTORY": "",
+ }
+}
+
+func tbCreateReaderRole(t *testing.T, ctx context.Context, admin *pgxpool.Pool, dsn string) (string, string) {
+ t.Helper()
+ random := make([]byte, 24)
+ if _, err := rand.Read(random); err != nil {
+ t.Fatal(err)
+ }
+ suffix := hex.EncodeToString(random)
+ role := "kg_three_reader_" + suffix[:12]
+ if _, err := admin.Exec(ctx, "CREATE ROLE "+pgx.Identifier{role}.Sanitize()+" LOGIN PASSWORD '"+suffix+"'"); err != nil {
+ t.Fatal("create isolated reader role: ", err)
+ }
+ t.Cleanup(func() {
+ cleanup := context.Background()
+ _, _ = admin.Exec(cleanup, "DROP OWNED BY "+pgx.Identifier{role}.Sanitize())
+ _, _ = admin.Exec(cleanup, "DROP ROLE "+pgx.Identifier{role}.Sanitize())
+ })
+ var schema string
+ if err := admin.QueryRow(ctx, "SELECT current_schema()").Scan(&schema); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := admin.Exec(ctx, "GRANT USAGE ON SCHEMA "+pgx.Identifier{schema}.Sanitize()+" TO "+pgx.Identifier{role}.Sanitize()); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := admin.Exec(ctx, "GRANT SELECT ON library_jobs,library_operations,library_attempts,library_versions,jobs TO "+pgx.Identifier{role}.Sanitize()); err != nil {
+ t.Fatal(err)
+ }
+ readerDSN, err := tbReaderDSN(dsn, role, suffix)
+ if err != nil {
+ t.Fatalf("build reader dsn: %v", err)
+ }
+ return readerDSN, role
+}
+
+// tbReaderDSN 显式从 URL 重建 reader 连接串。
+//
+// 不能用 pgxpool.Config.ConnString():pgx v5 保存的是解析时的原始连接串,
+// 只改 ConnConfig.User/Password 再取 ConnString 会原样返回 admin DSN,
+// 连接身份仍是 admin(上轮 gateway_control_reader_privileges_invalid 的根因)。
+func tbReaderDSN(dsn, user, password string) (string, error) {
+ parsed, err := url.Parse(dsn)
+ if err != nil {
+ return "", err
+ }
+ if parsed.Scheme != "postgres" && parsed.Scheme != "postgresql" {
+ return "", fmt.Errorf("reader dsn requires a postgres URL, got scheme %q", parsed.Scheme)
+ }
+ if parsed.Host == "" || strings.Trim(parsed.Path, "/") == "" {
+ return "", fmt.Errorf("reader dsn requires host and database")
+ }
+ parsed.User = url.UserPassword(user, password)
+ return parsed.String(), nil
+}
+
+// tbProbeReaderPrivileges 在实际连接上硬断言 reader 身份与只读权限;
+// 03-reader-privileges.json 只是证据,不通过断言就不启动网关。
+func tbProbeReaderPrivileges(t *testing.T, ctx context.Context, root, dsn, wantUser string) {
+ t.Helper()
+ if wantUser == "" {
+ t.Fatal("reader probe requires an explicit expected user")
+ }
+ parsed, err := pgxpool.ParseConfig(dsn)
+ if err != nil {
+ t.Fatalf("reader dsn parse: %v", err)
+ }
+ if parsed.ConnConfig.User != wantUser {
+ t.Fatalf("reader dsn identity user=%q, want %q", parsed.ConnConfig.User, wantUser)
+ }
+ pool, err := pgxpool.NewWithConfig(ctx, parsed)
+ if err != nil {
+ t.Fatalf("reader connect: %v", err)
+ }
+ defer pool.Close()
+ var user string
+ var super, createdb, createrole bool
+ if err := pool.QueryRow(ctx, `SELECT current_user, rolsuper, rolcreatedb, rolcreaterole FROM pg_roles WHERE rolname=current_user`).Scan(&user, &super, &createdb, &createrole); err != nil {
+ t.Fatalf("reader identity: %v", err)
+ }
+ privileges := map[string]any{"current_user": user, "rolsuper": super, "rolcreatedb": createdb, "rolcreaterole": createrole}
+ selectOKByTable := map[string]bool{}
+ dmlOKByTable := map[string]bool{}
+ for _, table := range []string{"library_jobs", "library_operations", "library_attempts", "library_versions", "jobs"} {
+ var selectOK, dmlOK bool
+ if err := pool.QueryRow(ctx, `SELECT has_table_privilege(current_user,$1,'SELECT'), has_table_privilege(current_user,$1,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')`, table).Scan(&selectOK, &dmlOK); err != nil {
+ t.Fatalf("reader privilege %s: %v", table, err)
+ }
+ selectOKByTable[table], dmlOKByTable[table] = selectOK, dmlOK
+ privileges[table] = map[string]bool{"select": selectOK, "dml": dmlOK}
+ }
+ tbWriteJSON(t, root, "03-reader-privileges.json", privileges)
+ if user != wantUser {
+ t.Fatalf("reader connection identity current_user=%q, want %q", user, wantUser)
+ }
+ if super || createdb || createrole {
+ t.Fatalf("reader role must not be super/createdb/createrole: %#v", privileges)
+ }
+ for _, table := range []string{"library_jobs", "library_operations", "library_attempts", "library_versions", "jobs"} {
+ if !selectOKByTable[table] || dmlOKByTable[table] {
+ t.Fatalf("reader role privilege mismatch on %s: select=%v dml=%v", table, selectOKByTable[table], dmlOKByTable[table])
+ }
+ }
+}
+
+// TestThreeBlockersReaderDSNIdentity 证明显式重建会切到 reader 身份并保留
+// host/database/query,且不会回带 admin 密码。真实链中的
+// 03-reader-privileges.json 断言同一 DSN 的实际会话身份。
+func TestThreeBlockersReaderDSNIdentity(t *testing.T) {
+ const adminDSN = "postgres://gov:admin-secret@db.internal:5432/kg_gov?sslmode=disable&application_name=three-blockers"
+ const role = "kg_three_reader_0123456789ab"
+ const password = "reader-secret"
+
+ readerDSN, err := tbReaderDSN(adminDSN, role, password)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(readerDSN, "admin-secret") {
+ t.Fatalf("reader dsn still carries the admin password")
+ }
+ parsed, err := url.Parse(readerDSN)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if parsed.User.Username() != role {
+ t.Fatalf("reader dsn user=%q, want %q", parsed.User.Username(), role)
+ }
+ if got, _ := parsed.User.Password(); got != password {
+ t.Fatalf("reader dsn password mismatch")
+ }
+ if parsed.Host != "db.internal:5432" || parsed.Path != "/kg_gov" {
+ t.Fatalf("reader dsn changed host/database: %s", readerDSN)
+ }
+ if parsed.Query().Get("sslmode") != "disable" || parsed.Query().Get("application_name") != "three-blockers" {
+ t.Fatalf("reader dsn dropped query: %s", readerDSN)
+ }
+ config, err := pgxpool.ParseConfig(readerDSN)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if config.ConnConfig.User != role || config.ConnConfig.Password != password ||
+ config.ConnConfig.Host != "db.internal" || config.ConnConfig.Port != 5432 || config.ConnConfig.Database != "kg_gov" {
+ t.Fatalf("pool config did not adopt reader identity: user=%q host=%q port=%d db=%q",
+ config.ConnConfig.User, config.ConnConfig.Host, config.ConnConfig.Port, config.ConnConfig.Database)
+ }
+ if _, err := tbReaderDSN("host=db.internal user=gov dbname=kg_gov", role, password); err == nil {
+ t.Fatal("keyword/value dsn must be rejected rather than silently keeping the admin identity")
+ }
+}
+
+func tbStartGateway(t *testing.T, ctx context.Context, env *tbEnv, readerDSN, targetID string) {
+ t.Helper()
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ evidence := filepath.Join(env.root, "05-identity.json")
+ raw := []byte(`{"kind":"isolated-three-blockers","writer":"writer","gateway_only_writer":true}`)
+ if err := os.WriteFile(evidence, raw, 0600); err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(raw)
+ digest := hex.EncodeToString(sum[:])
+ scope := objectgateway.Digest([]byte("minio\x00" + env.s3Endpoint + "\x00" + env.bucket))
+ payload, err := json.Marshal(objectgateway.Attestation{Protocol: objectgateway.Protocol, Scope: scope,
+ GatewayID: env.project, IssuedAt: time.Now().Add(-time.Second), ExpiresAt: time.Now().Add(50 * time.Minute),
+ WriterInventory: digest, PolicyEvidence: digest, DenialEvidence: digest, DrainEvidence: digest})
+ if err != nil {
+ t.Fatal(err)
+ }
+ signed, err := json.Marshal(objectgateway.SignedAttestation{Payload: payload, Signature: ed25519.Sign(priv, payload)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ audit := filepath.Join(env.root, "audit.json")
+ if err := os.WriteFile(audit, signed, 0600); err != nil {
+ t.Fatal(err)
+ }
+ tbStart(t, env.root, "gateway", filepath.Join(env.bin, "object-gateway"), map[string]string{
+ "OBJECT_GATEWAY_LISTEN": env.gatewayAddr, "OBJECT_GATEWAY_DATABASE_URL": tbGetenvID("KG_THREE_GOVERNANCE_DSN"),
+ "OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL": readerDSN, "OBJECT_GATEWAY_ARCHIVE_ROOT": env.archiveRoot,
+ "OBJECT_GATEWAY_TARGET_ID": targetID, "OBJECT_GATEWAY_PROVIDER": "minio",
+ "OBJECT_GATEWAY_ENDPOINT": env.s3Endpoint, "OBJECT_GATEWAY_BUCKET": env.bucket,
+ "OBJECT_GATEWAY_REGION": "us-east-1", "OBJECT_GATEWAY_ACCESS_KEY": env.s3Access,
+ "OBJECT_GATEWAY_SECRET_KEY": env.s3Secret, "OBJECT_GATEWAY_DATA_TOKEN": tbGetenvID("KG_THREE_DATA_TOKEN"),
+ "OBJECT_GATEWAY_CONTROL_TOKEN": tbGetenvID("KG_THREE_CONTROL_TOKEN"), "OBJECT_GATEWAY_ID": env.project,
+ "OBJECT_GATEWAY_AUDIT_FILE": audit, "OBJECT_GATEWAY_AUDIT_PUBLIC_KEY": base64.StdEncoding.EncodeToString(pub),
+ })
+ client, err := objectgateway.NewClient("http://"+env.gatewayAddr, tbGetenvID("KG_THREE_DATA_TOKEN"), scope)
+ if err != nil {
+ t.Fatal(err)
+ }
+ deadline := time.Now().Add(60 * time.Second)
+ for {
+ if err := client.ReadyForAcquisition(ctx); err == nil {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("object gateway never became ready (see gateway.log)")
+ }
+ time.Sleep(200 * time.Millisecond)
+ }
+}
+
+// tbLogin 走真实 API 登录(一次性治理库里的 admin),保留 cookie。
+func tbLogin(t *testing.T, ctx context.Context, gov *pgxpool.Pool, base string) *tbAPI {
+ t.Helper()
+ password := hex.EncodeToString(func() []byte { b := make([]byte, 18); _, _ = rand.Read(b); return b }())
+ hash, err := bcrypt.GenerateFromPassword([]byte(password), 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := gov.Exec(ctx, `UPDATE users SET password_hash=$1, must_change_password=false, failed_attempts=0,
+ locked_until=NULL, state='active' WHERE username='admin'`, string(hash)); err != nil {
+ t.Fatal(err)
+ }
+ jar, err := cookiejar.New(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ api := &tbAPI{base: base, client: &http.Client{Jar: jar, Timeout: 30 * time.Second}}
+ status, body := api.do(t, http.MethodPost, "/api/v1/auth/login", map[string]string{"username": "admin", "password": password})
+ if status != http.StatusOK {
+ t.Fatalf("admin login => %d %v", status, body)
+ }
+ return api
+}
+
+type tbAPI struct {
+ base string
+ client *http.Client
+}
+
+func (a *tbAPI) do(t *testing.T, method, path string, payload any) (int, map[string]any) {
+ t.Helper()
+ var reader io.Reader
+ if payload != nil {
+ raw, err := json.Marshal(payload)
+ if err != nil {
+ t.Fatal(err)
+ }
+ reader = bytes.NewReader(raw)
+ }
+ req, err := http.NewRequest(method, a.base+path, reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if payload != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ resp, err := a.client.Do(req)
+ if err != nil {
+ t.Fatalf("%s %s: %v", method, path, err)
+ }
+ defer resp.Body.Close()
+ raw, _ := io.ReadAll(resp.Body)
+ var body map[string]any
+ _ = json.Unmarshal(raw, &body)
+ return resp.StatusCode, body
+}
+
+func tbEnqueueVerify(t *testing.T, api *tbAPI) governance.LibraryJob {
+ t.Helper()
+ status, body := api.do(t, http.MethodPost, "/api/v1/library/verifications", map[string]string{"scope": "internal"})
+ if status != http.StatusAccepted {
+ t.Fatalf("enqueue verify => %d %v", status, body)
+ }
+ raw, _ := json.Marshal(body["job"])
+ var job governance.LibraryJob
+ if err := json.Unmarshal(raw, &job); err != nil {
+ t.Fatalf("decode accepted job: %v (%v)", err, body)
+ }
+ if job.ID == "" {
+ t.Fatalf("accepted job missing id: %v", body)
+ }
+ return job
+}
+
+func tbPostRecover(t *testing.T, api *tbAPI, windowJobID string) jobs.Job {
+ t.Helper()
+ status, body := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
+ map[string]string{"scope": "internal", "window_job_id": windowJobID})
+ if status != http.StatusAccepted {
+ t.Fatalf("recover enqueue => %d %v", status, body)
+ }
+ raw, _ := json.Marshal(body["job"])
+ var job jobs.Job
+ if err := json.Unmarshal(raw, &job); err != nil {
+ t.Fatalf("decode recovery queue job: %v", err)
+ }
+ return job
+}
+
+func tbClaimQueue(t *testing.T, ctx context.Context, store *postgresstore.Store, kind string) *jobs.Job {
+ t.Helper()
+ deadline := time.Now().Add(30 * time.Second)
+ for {
+ job, err := store.Claim(ctx, tbGetenvID("KG_THREE_WORKER_ID"), []string{kind}, time.Now().UTC(), 2*time.Minute)
+ if err == nil && job != nil {
+ return job
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("claim %s queue job: %v", kind, err)
+ }
+ time.Sleep(200 * time.Millisecond)
+ }
+}
+
+// tbWaitQueueTerminal 等同一消费外壳把 jobs 行写回终态后按 id 读回;
+// 只读,不修改队列状态。
+func tbWaitQueueTerminal(t *testing.T, ctx context.Context, store *postgresstore.Store, id string) jobs.Job {
+ t.Helper()
+ deadline := time.Now().Add(2 * time.Minute)
+ var last error
+ for {
+ list, err := store.ListByType(ctx, "library.recover", 200)
+ if err == nil {
+ for _, job := range list {
+ if job.ID == id {
+ if job.State.Terminal() {
+ return job
+ }
+ break
+ }
+ }
+ } else {
+ last = err
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("recovery queue %s never reached terminal state: %v", id, last)
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+}
+
+// tbUnrelatedSentinel 读回无关租户的行身份,证明恢复没有碰不属于本次窗口的数据。
+func tbUnrelatedSentinel(t *testing.T, ctx context.Context, pool *pgxpool.Pool) string {
+ t.Helper()
+ ids := func(table string) string {
+ rows, err := pool.Query(ctx, `SELECT id FROM `+table+` WHERE tenant_id=31999 ORDER BY id`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer rows.Close()
+ var out []string
+ for rows.Next() {
+ var id string
+ if err := rows.Scan(&id); err != nil {
+ t.Fatal(err)
+ }
+ out = append(out, id)
+ }
+ if err := rows.Err(); err != nil {
+ t.Fatal(err)
+ }
+ return strings.Join(out, ",")
+ }
+ return ids("knowledges") + "/" + ids("chunks")
+}
+
+func tbWaitStage(t *testing.T, ctx context.Context, env *tbEnv, jobID, stage string, timeout time.Duration) {
+ t.Helper()
+ deadline := time.Now().Add(timeout)
+ for time.Now().Before(deadline) {
+ var state string
+ err := env.gov.QueryRow(ctx, `SELECT state FROM library_job_stages WHERE job_id=$1 AND stage=$2`, jobID, stage).Scan(&state)
+ if err == nil && (state == "running" || state == "succeeded") {
+ return
+ }
+ var jobState string
+ _ = env.gov.QueryRow(ctx, `SELECT state FROM library_jobs WHERE id=$1`, jobID).Scan(&jobState)
+ if jobState == "failed" || jobState == "succeeded" {
+ t.Fatalf("job %s reached %s before stage %s", jobID, jobState, stage)
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ t.Fatalf("job %s never reached stage %s", jobID, stage)
+}
+
+// tbRefusingProxy 只在传输层拒绝指定的 ES 请求(不转发),其余原样转发。
+type tbRefusingProxy struct {
+ url string
+ refuse atomic.Bool
+ refused atomic.Int64
+}
+
+func tbNewRefusingProxy(t *testing.T, env *tbEnv, upstream string) *tbRefusingProxy {
+ t.Helper()
+ p := &tbRefusingProxy{}
+ server := &http.Server{}
+ _ = server
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ raw, _ := io.ReadAll(r.Body)
+ if p.refuse.Load() && r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/_bulk") {
+ p.refused.Add(1)
+ http.Error(w, "injected ES restore refusal before dispatch", http.StatusServiceUnavailable)
+ return
+ }
+ req, err := http.NewRequestWithContext(r.Context(), r.Method, upstream+r.URL.RequestURI(), bytes.NewReader(raw))
+ if err != nil {
+ http.Error(w, "proxy request failed", 500)
+ return
+ }
+ req.Header = r.Header.Clone()
+ req.Header.Del("Accept-Encoding")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ http.Error(w, "upstream failed", 502)
+ return
+ }
+ body, _ := io.ReadAll(resp.Body)
+ resp.Body.Close()
+ for k, values := range resp.Header {
+ for _, v := range values {
+ w.Header().Add(k, v)
+ }
+ }
+ w.WriteHeader(resp.StatusCode)
+ _, _ = w.Write(body)
+ })
+ listener, err := tbListenLoopback()
+ if err != nil {
+ t.Fatal(err)
+ }
+ srv := &http.Server{Handler: handler}
+ go func() { _ = srv.Serve(listener) }()
+ t.Cleanup(func() { _ = srv.Close() })
+ p.url = "http://" + listener.Addr().String()
+ return p
+}
+
+func (p *tbRefusingProxy) RefuseNextBulk() { p.refuse.Store(true) }
+func (p *tbRefusingProxy) Disarm() { p.refuse.Store(false) }
+
+func tbDump(t *testing.T, ctx context.Context, pool *pgxpool.Pool) map[string][]string {
+ t.Helper()
+ dump := map[string][]string{}
+ tables := []string{"knowledges", "chunks", "knowledge_tags", "knowledge_tag_relations", "resources",
+ "resource_bindings", "wiki_folders", "wiki_pages", "wiki_page_issues", "wiki_log_entries"}
+ for _, table := range tables {
+ rows, err := pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table))
+ if err != nil {
+ t.Fatal(err)
+ }
+ values := []string{}
+ for rows.Next() {
+ var raw string
+ if err := rows.Scan(&raw); err != nil {
+ rows.Close()
+ t.Fatal(err)
+ }
+ values = append(values, raw)
+ }
+ rows.Close()
+ dump[table] = values
+ }
+ return dump
+}
+
+func tbDiffDump(before, after map[string][]string) string {
+ tables := make([]string, 0, len(before))
+ for table := range before {
+ tables = append(tables, table)
+ }
+ sort.Strings(tables)
+ for _, table := range tables {
+ if len(before[table]) != len(after[table]) {
+ return fmt.Sprintf("%s rows %d -> %d", table, len(before[table]), len(after[table]))
+ }
+ for i := range before[table] {
+ if before[table][i] != after[table][i] {
+ return fmt.Sprintf("%s row %d changed: %s", table, i, after[table][i])
+ }
+ }
+ }
+ return ""
+}
+
+func tbIndexReadback(t *testing.T, ctx context.Context, env *tbEnv) map[string]string {
+ t.Helper()
+ req, _ := http.NewRequestWithContext(ctx, http.MethodPost,
+ fmt.Sprintf("%s/%s/_search?size=100", tbGetenv(t, "KG_LIBRARY_TEST_ELASTIC_URL"), env.index),
+ strings.NewReader(`{"query":{"match_all":{}},"size":100,"_source":true}`))
+ req.Header.Set("Content-Type", "application/json")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ raw, _ := io.ReadAll(resp.Body)
+ var parsed struct {
+ Hits struct {
+ Hits []struct {
+ ID string `json:"_id"`
+ Source json.RawMessage `json:"_source"`
+ } `json:"hits"`
+ } `json:"hits"`
+ }
+ if err := json.Unmarshal(raw, &parsed); err != nil || resp.StatusCode != 200 {
+ t.Fatalf("index readback %d %s", resp.StatusCode, raw)
+ }
+ result := map[string]string{}
+ for _, hit := range parsed.Hits.Hits {
+ var source map[string]any
+ if err := json.Unmarshal(hit.Source, &source); err != nil {
+ t.Fatalf("index readback source %s: %v", hit.ID, err)
+ }
+ canonical, err := json.Marshal(source)
+ if err != nil {
+ t.Fatalf("index readback canonical %s: %v", hit.ID, err)
+ }
+ result[hit.ID] = string(canonical)
+ }
+ return result
+}
+
+func tbDiffIndex(before, after map[string]string) string {
+ keys := map[string]bool{}
+ for k := range before {
+ keys[k] = true
+ }
+ for k := range after {
+ keys[k] = true
+ }
+ ordered := make([]string, 0, len(keys))
+ for k := range keys {
+ ordered = append(ordered, k)
+ }
+ sort.Strings(ordered)
+ for _, k := range ordered {
+ if before[k] != after[k] {
+ return fmt.Sprintf("document %s changed: %q -> %q", k, before[k], after[k])
+ }
+ }
+ return ""
+}
+
+func tbIngressHealthy(t *testing.T, env *tbEnv) error {
+ t.Helper()
+ req, _ := http.NewRequest(http.MethodGet, env.ingress, nil)
+ req.Close = true
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ _, _ = io.Copy(io.Discard, resp.Body)
+ if resp.StatusCode != 200 {
+ return fmt.Errorf("ingress HTTP %d", resp.StatusCode)
+ }
+ return nil
+}
+
+func tbAssertIngressHealthy(t *testing.T, env *tbEnv) {
+ t.Helper()
+ if err := tbIngressHealthy(t, env); err != nil {
+ t.Fatalf("ingress not restored: %v", err)
+ }
+}
+
+func tbIngressClosed(env *tbEnv) error {
+ req, _ := http.NewRequest(http.MethodGet, env.ingress, nil)
+ req.Close = true
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil
+ }
+ defer resp.Body.Close()
+ return fmt.Errorf("ingress still answers HTTP %d", resp.StatusCode)
+}
+
+func tbAssertMaintenanceReleased(t *testing.T, ctx context.Context, env *tbEnv) {
+ t.Helper()
+ inst, err := env.store.InstanceMaintenanceOf(ctx, env.project)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if inst == nil || inst.ReleasedAt == nil {
+ t.Fatalf("instance maintenance not released: %#v", inst)
+ }
+ var released *time.Time
+ var state string
+ if err := env.gov.QueryRow(ctx, `SELECT released_at,state FROM library_maintenance WHERE scope='internal'`).Scan(&released, &state); err != nil {
+ t.Fatal(err)
+ }
+ if released == nil {
+ t.Fatalf("scope maintenance not released: state=%s", state)
+ }
+}
+
+func tbFindDrillLedger(t *testing.T, ctx context.Context, env *tbEnv, jobID string) libraryrelease.WriteSet {
+ t.Helper()
+ var digest string
+ if err := env.gov.QueryRow(ctx, `SELECT evidence->>'writeset_digest' FROM library_versions
+ WHERE source_job_id=$1 AND kind='production_prebackup'`, jobID).Scan(&digest); err != nil {
+ t.Fatal(err)
+ }
+ if digest == "" {
+ t.Fatal("no persisted drill writeset digest")
+ }
+ set, err := v071.LoadWriteSet(env.archiveRoot, digest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return *set
+}
+
+func tbListenLoopback() (net.Listener, error) {
+ return net.Listen("tcp", "127.0.0.1:0")
+}
diff --git a/internal/httpapi/library_verify.go b/internal/httpapi/library_verify.go
index 1e72d9c..a742ab6 100644
--- a/internal/httpapi/library_verify.go
+++ b/internal/httpapi/library_verify.go
@@ -97,11 +97,25 @@ func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config)
return
}
now := time.Now().UTC()
+ // 验证作业必须有和同步/恢复同一套不可变 operation:worker 的 owned attempt 从
+ // job.operation_id 取身份,恢复派发闸口和对象网关都按它核验。没有 operation 的
+ // verify 作业在真实恢复时会被 fail closed(审核 B2-R1)。
+ intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobVerify,
+ "", digest, identity.Username, "", now)
+ if intentErr != nil {
+ if isIntentConflict(intentErr) {
+ writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
+ "error": "同一幂等键对应了不同的请求"})
+ return
+ }
+ writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify operation"})
+ return
+ }
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
Kind: governance.LibraryJobVerify,
- IdempotencyKey: "library.verify:" + request.Scope + ":" + now.Format("20060102T150405.000000000"),
- RequestDigest: digest, Actor: identity.Username,
+ IdempotencyKey: intent.JobKey,
+ RequestDigest: digest, OperationID: intent.Operation.ID, Actor: identity.Username,
}, governance.VerifyStages(), false, now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active",
diff --git a/internal/httpapi/library_verify_test.go b/internal/httpapi/library_verify_test.go
index 0e15593..a34d8ba 100644
--- a/internal/httpapi/library_verify_test.go
+++ b/internal/httpapi/library_verify_test.go
@@ -99,6 +99,14 @@ func TestLibraryVerifyEnqueuesVerifyJobWithCanaryStages(t *testing.T) {
if !strings.HasPrefix(store.lastJob.ID, "libverify-internal-") {
t.Fatalf("job id=%q", store.lastJob.ID)
}
+ // B2-R1:verify 作业必须带不可变 operation,worker 的 owned attempt / 恢复派发
+ // 身份都从它取;没有 operation 的作业在真实恢复时会被 fail closed。
+ if store.lastJob.OperationID == "" || store.lastJob.OperationID != store.lastOperation.ID {
+ t.Fatalf("verify job must carry its operation: job=%#v op=%#v", store.lastJob, store.lastOperation)
+ }
+ if store.lastOperation.Kind != governance.LibraryJobVerify || store.lastOperation.Scope != "internal" {
+ t.Fatalf("verify operation identity=%#v", store.lastOperation)
+ }
// 阶段必须包含 canary:没有"写 canary 证明校验有效"这一步,验证就只是自证。
want := governance.VerifyStages()
if len(want) != 5 || want[2] != "canary" || want[3] != "restore" || want[4] != "finalize" {
diff --git a/internal/libraryrelease/writeset.go b/internal/libraryrelease/writeset.go
index 7ef2620..8a4c260 100644
--- a/internal/libraryrelease/writeset.go
+++ b/internal/libraryrelease/writeset.go
@@ -19,6 +19,12 @@ const WriteSetVersion = 1
// MigrationWriteSetVersion rejects reinterpretation by old V1 executors.
const MigrationWriteSetVersion = 2
+// WriteSetKindRestoreDrill marks a write set that authorizes only a recovery
+// drill: canary rows plus a zero-object/zero-index execution plan. It shares
+// the V2 envelope so the object gateway controller can verify it, but it is
+// never a publish plan and must not be adopted as one.
+const WriteSetKindRestoreDrill = "restore-drill"
+
// WriteSetRow 是台账里的一行:本次发布明确要写的精确身份 + 写前状态。
//
// 为什么需要"写前状态":恢复不能在目标里按"没人引用"推断残渣。历史孤儿资源、
@@ -89,6 +95,10 @@ type LocalIDMapping struct {
}
type WriteSet struct {
+ // Kind 区分发布执行计划与受限演练恢复计划。空值 = 历史发布计划。
+ // 演练计划必须显式为 restore-drill:它只授权 canary 回放,不含对象/索引
+ // 写项,读取方不能把它当发布计划接管。
+ Kind string `json:"kind,omitempty"`
Index []IndexWrite `json:"index,omitempty"`
Objects []ObjectWrite `json:"objects,omitempty"`
SchemaDigest string `json:"schema_digest,omitempty"`
@@ -218,6 +228,22 @@ func (w *WriteSet) Verify() error {
if w.Version != WriteSetVersion && w.Version != MigrationWriteSetVersion {
return fmt.Errorf("写集合记录版本 %d 不受支持(当前 %d)", w.Version, WriteSetVersion)
}
+ if w.Kind != "" && w.Kind != WriteSetKindRestoreDrill {
+ return fmt.Errorf("写集合记录种类 %q 不受支持", w.Kind)
+ }
+ if w.Kind == WriteSetKindRestoreDrill {
+ // 演练恢复计划只回放写前镜像:不新建对象、不按发布计划改索引、不做本地 ID 映射。
+ // 任何一项非空都说明它被当成发布计划构造/改写过,拒绝。
+ if w.Version != MigrationWriteSetVersion || w.ContractVersion != 1 || len(w.SchemaDigest) != 64 {
+ return errors.New("演练恢复计划缺少可执行契约(V2 + contract 1 + schema 摘要)")
+ }
+ if len(w.Objects) != 0 || len(w.Index) != 0 || len(w.LocalIDs) != 0 {
+ return errors.New("演练恢复计划不得包含对象/索引/本地映射写项")
+ }
+ if w.SourceTenantID == 0 || w.SourceKnowledgeBaseID == "" {
+ return errors.New("演练恢复计划缺少来源身份")
+ }
+ }
if w.JobID == "" || w.BackupSnapshotID == "" || w.TargetSnapshotID == "" {
return errors.New("写集合记录缺少作业/预备份/快照绑定")
}
@@ -248,6 +274,9 @@ func (w *WriteSet) Verify() error {
return nil
}
+// IsRestoreDrill 表示这是一份受限演练恢复计划,而不是发布执行计划。
+func (w WriteSet) IsRestoreDrill() bool { return w.Kind == WriteSetKindRestoreDrill }
+
// CreatedIDs 返回"本次新建"的精确身份(写前不存在的行)。恢复只允许删这些。
func (w WriteSet) CreatedIDs(table string) []string {
ids := make([]string, 0, 8)
diff --git a/internal/libraryverify/canary_test.go b/internal/libraryverify/canary_test.go
index af59222..c50f957 100644
--- a/internal/libraryverify/canary_test.go
+++ b/internal/libraryverify/canary_test.go
@@ -23,7 +23,7 @@ func TestCanaryPersistsAndRegistersEvidenceBeforeWrite(t *testing.T) {
root := t.TempDir()
ledger := drillLedger()
registered, marked, written := false, false, false
- err := writeDrillCanary(context.Background(), root, &ledger, "canary-id",
+ err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", nil,
func(context.Context) error {
if !registered {
t.Fatal("risk marker preceded durable registration")
@@ -67,7 +67,7 @@ func TestCanaryNeverWritesWithoutDurableLedger(t *testing.T) {
}
ledger := drillLedger()
marked, written := false, false
- err := writeDrillCanary(context.Background(), root, &ledger, "canary-id",
+ err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", nil,
func(context.Context) error { marked = true; return nil },
func(context.Context) error { marked = true; return nil },
func(context.Context) error { written = true; return nil })
@@ -80,7 +80,7 @@ func TestCanaryNeverWritesWithoutDurableLedger(t *testing.T) {
func TestCanaryNeverWritesWhenRegistrationFails(t *testing.T) {
ledger := drillLedger()
marked, written := false, false
- err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id",
+ err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", nil,
func(context.Context) error { marked = true; return nil },
func(context.Context) error { return errors.New("governance registration unavailable") },
func(context.Context) error { written = true; return nil })
@@ -92,7 +92,7 @@ func TestCanaryNeverWritesWhenRegistrationFails(t *testing.T) {
func TestCanaryNeverWritesWhenRiskMarkerFails(t *testing.T) {
ledger := drillLedger()
written := false
- err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id",
+ err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", nil,
func(context.Context) error { return errors.New("journal unavailable") },
func(context.Context) error { return nil },
func(context.Context) error { written = true; return nil })
diff --git a/internal/libraryverify/verify.go b/internal/libraryverify/verify.go
index 34d9d02..0b90dc8 100644
--- a/internal/libraryverify/verify.go
+++ b/internal/libraryverify/verify.go
@@ -252,6 +252,11 @@ func Run(ctx context.Context, config appconfig.Config, scope, jobID string, opts
report.Canary = canary
progress.report("canary", "running", "写入一个隔离 canary,验证校验能发现差异")
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
+ func(set *libraryrelease.WriteSet) error {
+ // 计划必须在 Seal/落盘/登记之前定型:对象网关控制器按登记摘要加载,
+ // 这里补 V2 信封、目标 schema、来源身份与当前 owned attempt 绑定。
+ return target.PrepareDrillPlan(ctx, set)
+ },
func(ctx context.Context) error {
if err := window.MarkContentRisk(ctx); err != nil {
return err
@@ -397,8 +402,14 @@ func ledgerClone(set *libraryrelease.WriteSet) *libraryrelease.WriteSet {
// writeDrillCanary 是"写入前先留痕"的唯一实现:seal → 落盘 → 登记 → 标风险 → 写。
// 任何一步失败都不写目标;登记在标风险之前,恢复入口才拿得到这份写集合。
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string,
+ prepare func(*libraryrelease.WriteSet) error,
mark func(context.Context) error, record func(context.Context) error, write func(context.Context) error) error {
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
+ if prepare != nil {
+ if err := prepare(ledger); err != nil {
+ return err
+ }
+ }
if err := ledger.Seal(); err != nil {
return err
}
diff --git a/internal/releaseadapter/v071/gateway_object_store.go b/internal/releaseadapter/v071/gateway_object_store.go
index b41ddc6..6edc48d 100644
--- a/internal/releaseadapter/v071/gateway_object_store.go
+++ b/internal/releaseadapter/v071/gateway_object_store.go
@@ -53,6 +53,12 @@ func ObjectGatewayPlan(set libraryrelease.WriteSet) (objectgateway.Plan, error)
func gatewayCreateOperation(plan, owner string) string {
return objectgateway.Digest([]byte(plan + "\x00" + owner + "\x00create"))
}
+
+// OwnedAttemptClaim 暴露本装配冻结的 attempt claim:受限演练计划/回放必须与它一致。
+func (s *GatewayObjectStore) OwnedAttemptClaim() (objectgateway.Claim, bool) {
+ return s.Claim, s.Automatic
+}
+
func (s *GatewayObjectStore) BindObjectPlan(set libraryrelease.WriteSet) error {
if !validLowerSHA256(set.Digest) || set.JobID == "" {
return errors.New("gateway frozen plan identity missing")
diff --git a/internal/releaseadapter/v071/library_production_target.go b/internal/releaseadapter/v071/library_production_target.go
index d37b275..4c48765 100644
--- a/internal/releaseadapter/v071/library_production_target.go
+++ b/internal/releaseadapter/v071/library_production_target.go
@@ -19,6 +19,7 @@ import (
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
+ "knowledge-governance/internal/objectgateway"
"knowledge-governance/internal/testbarrier"
)
@@ -116,6 +117,9 @@ type ProductionLibraryTarget struct {
preserveChunkSequence bool
activePlan *libraryrelease.WriteSet
activeSnapshot string
+ // drillRestore 表示 activePlan 是受限演练恢复计划:索引按写前镜像重建,
+ // 对象只做只读字节核对(计划本身零对象写项)。发布恢复永不为 true。
+ drillRestore bool
// restoreScope / restoreSourceSnapshot / restoreBackupSnapshot 在 Restore 进入时
// 从**原持久锚点**冻结:scope 来自真实装配,来源快照与写前镜像来自本次 WriteSet
// 与原 backup 引用。它们不能由调用方在恢复中途改写,也不允许同一个快照既当
@@ -681,7 +685,7 @@ WHERE tenant_id=$1 AND id = ANY($2::text[])`, t.scope.TenantID, ids)
}
func (t *ProductionLibraryTarget) importIndex(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
- if t.activePlan != nil {
+ if t.activePlan != nil && !t.drillRestore {
if t.preserveChunkSequence {
return t.restorePlannedIndex(ctx)
}
@@ -927,6 +931,10 @@ func (t *ProductionLibraryTarget) AdoptWriteSet(set libraryrelease.WriteSet) err
if err := set.Verify(); err != nil {
return fmt.Errorf("写集合记录校验失败:%w", err)
}
+ if set.IsRestoreDrill() {
+ // 演练计划只描述 canary 回放,不是发布执行计划;发布恢复入口不得消费它。
+ return errors.New("演练恢复计划不能作为发布执行计划接管")
+ }
if set.TenantID != t.scope.TenantID || set.KnowledgeBaseID != t.scope.KnowledgeBaseID {
return fmt.Errorf("写集合绑定的是 %d/%s,与目标 %d/%s 不一致",
set.TenantID, set.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID)
@@ -983,6 +991,89 @@ func ValidateDrillWriteSet(set libraryrelease.WriteSet) error {
return nil
}
+// PrepareDrillPlan 把演练台账升级为**受限可执行计划**:V2 执行信封 + 目标 schema
+// 摘要 + 真实来源身份 + 当前 owned attempt 绑定,但对象/索引/本地映射写项必须为空。
+//
+// 为什么必须在这里而不是在 Apply 里临时补:对象网关控制器只认治理库里
+// production_prebackup 证据登记的 writeset_digest;计划必须在 Seal/落盘/登记之前
+// 定型,否则控制器加载到的摘要是另一份形状。它只增加受限语义,不放宽任何发布校验。
+func (t *ProductionLibraryTarget) PrepareDrillPlan(ctx context.Context, set *libraryrelease.WriteSet) error {
+ if set == nil || set.JobID == "" {
+ return errors.New("演练恢复计划缺少作业身份")
+ }
+ if set.TenantID != t.scope.TenantID || set.KnowledgeBaseID != t.scope.KnowledgeBaseID {
+ return fmt.Errorf("演练恢复计划绑定的是 %d/%s,与目标 %d/%s 不一致",
+ set.TenantID, set.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID)
+ }
+ if err := set.ValidateRestoreBinding(set.BackupSnapshotID, t.scope.TenantID, t.scope.KnowledgeBaseID); err != nil {
+ return err
+ }
+ archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
+ if err != nil {
+ return errors.New("归档不可用")
+ }
+ source, err := archive.Load(set.TargetSnapshotID)
+ archive.Close()
+ if err != nil {
+ return errors.New("演练来源快照不可读")
+ }
+ if source.TenantID != t.scope.TenantID || source.KnowledgeBaseID != t.scope.KnowledgeBaseID {
+ return errors.New("演练来源快照不属于当前目标范围")
+ }
+ _, schemaDigest, err := t.migrationCatalog(ctx)
+ if err != nil {
+ return err
+ }
+ // 先清空任何发布写项:演练计划只允许 canary 行。
+ set.Kind = libraryrelease.WriteSetKindRestoreDrill
+ set.Version = libraryrelease.MigrationWriteSetVersion
+ set.ContractVersion = migrationIdentityContractVersion
+ set.SourceTenantID, set.SourceKnowledgeBaseID = source.TenantID, source.KnowledgeBaseID
+ set.SchemaDigest = schemaDigest
+ set.Index, set.Objects, set.LocalIDs = nil, nil, nil
+ // 自动对象网关没有静态 grant:计划签发必须绑定当前 owned attempt,
+ // 网关控制器会再按治理库核一次(不靠调用方自证)。
+ if claim, automatic := t.ownedAttempt(); automatic {
+ return validateDrillAttempt(claim, set.JobID)
+ }
+ return nil
+}
+
+// ownedAttempt 返回本目标装配时冻结的对象网关 claim。只有 Automatic 网关需要它。
+func (t *ProductionLibraryTarget) ownedAttempt() (objectgateway.Claim, bool) {
+ if provider, ok := t.config.Objects.(interface {
+ OwnedAttemptClaim() (objectgateway.Claim, bool)
+ }); ok {
+ return provider.OwnedAttemptClaim()
+ }
+ return objectgateway.Claim{}, false
+}
+
+// validateDrillAttempt 要求计划/回放绑定一个真实的作业代次,而不是调用方编造的身份。
+func validateDrillAttempt(claim objectgateway.Claim, jobID string) error {
+ if claim.JobID == "" || claim.JobID != jobID || claim.OperationID == "" || claim.Owner == "" || claim.AttemptEpoch < 1 {
+ return errors.New("演练恢复缺少当前 owned attempt 身份(job/operation/owner/epoch)")
+ }
+ return nil
+}
+
+// validateDrillDispatch 证明这次演练的派发身份来自原窗口的真实 attempt 与恢复投递,
+// 不是把 gate 置 nil 绕过 DS01。
+func (t *ProductionLibraryTarget) validateDrillDispatch(claim objectgateway.Claim, automatic bool) error {
+ config := t.config
+ if config.RestoreTargetIdentity == "" || config.RestoreScope == "" || config.RestoreInstance == "" ||
+ config.RestoreOperationID == "" || config.RestoreWindowJobID == "" || config.RestoreAttemptOwner == "" ||
+ config.RestoreAttemptEpoch < 1 || config.RestoreRecoveryJobID == "" || config.RestoreRecoveryOperationID == "" {
+ return errors.New("演练恢复缺少完整的派发身份(目标/scope/实例/原 operation/窗口/attempt/恢复作业)")
+ }
+ if automatic && (claim.JobID != config.RestoreWindowJobID || claim.OperationID != config.RestoreOperationID ||
+ claim.Owner != config.RestoreAttemptOwner || claim.AttemptEpoch != config.RestoreAttemptEpoch ||
+ claim.RecoveryJobID != config.RestoreRecoveryJobID || claim.RecoveryOperationID != config.RestoreRecoveryOperationID) {
+ return errors.New("演练恢复的派发身份与当前 owned attempt 不一致")
+ }
+ return nil
+}
+
// RestoreDrill 是恢复演练**限定范围**的安全恢复入口。
//
// 为什么不是直接调用 Restore:Restore 的写集合是"发布执行计划"(v2 合同 + 索引/对象/
@@ -990,7 +1081,10 @@ func ValidateDrillWriteSet(set libraryrelease.WriteSet) error {
// 一份从未发生过的发布。这里保留同一份 DS01 身份合同(来源快照 ≠ 写前镜像、同一作业/
// 目标/预备份)与同一批 Apply 原语:四个阶段都按写前镜像**整段回放**(先清 scope 再
// 插入归档行),不是"只删 canary";回放后再用整实例指纹证明目标与预备份一致。
-// 有派发闸口的目标不走这条入口:那条路径必须有过派发证据的恢复计划。
+//
+// 自动对象网关保持启用:演练计划显式零对象写项,仍由控制器按当前 attempt/job/目标/
+// 写前镜像签发零写入许可;对象阶段只做只读字节核对,任何创建/删除都会被网关拒绝。
+// 有派发闸口的目标也不再把 gate 置 nil:必须携带原窗口 attempt + 恢复投递身份。
func (t *ProductionLibraryTarget) RestoreDrill(ctx context.Context, backup libraryrelease.BackupRef) error {
if backup.SnapshotID == "" {
return errors.New("恢复需要绑定的预备份快照")
@@ -1007,8 +1101,43 @@ func (t *ProductionLibraryTarget) RestoreDrill(ctx context.Context, backup libra
if err := ValidateDrillWriteSet(*backup.WriteSet); err != nil {
return err
}
+ if !backup.WriteSet.IsRestoreDrill() {
+ return errors.New("演练恢复只接受受限恢复计划(restore-drill),拒绝发布执行计划")
+ }
+ claim, automatic := t.ownedAttempt()
+ if automatic {
+ if err := validateDrillAttempt(claim, backup.WriteSet.JobID); err != nil {
+ return err
+ }
+ }
if t.config.RestoreDispatch != nil {
- return errors.New("目标配置了恢复派发闸口:演练不能用绕过派发证据的回放入口")
+ if backup.WriteSet.Target != t.config.RestoreScope {
+ return fmt.Errorf("演练恢复计划 scope=%s,与派发身份 scope=%s 不一致",
+ backup.WriteSet.Target, t.config.RestoreScope)
+ }
+ if err := t.validateDrillDispatch(claim, automatic); err != nil {
+ return err
+ }
+ }
+ // 对象网关:冻结零对象计划。控制器仍按治理库核 attempt/job/目标/写前镜像;
+ // 没有冻结计划时 EnsureObjectGrant 会继续 fail closed。
+ if binder, ok := t.config.Objects.(interface {
+ BindObjectPlan(libraryrelease.WriteSet) error
+ }); ok {
+ if err := binder.BindObjectPlan(*backup.WriteSet); err != nil {
+ return err
+ }
+ }
+ adopted := *backup.WriteSet
+ adopted.Rows = append([]libraryrelease.WriteSetRow(nil), backup.WriteSet.Rows...)
+ t.activePlan = &adopted
+ t.drillRestore = true
+ // DS01 准入必须在任何 PG/对象/索引写之前:原窗口若还有未决派发记录,
+ // 即使计划是零对象也要 held。
+ if t.config.RestoreDispatch != nil {
+ if err := t.admitRestorePlan(ctx); err != nil {
+ return err
+ }
}
// 幂等:恢复收尾可能重放(上次恢复到一半崩溃)。这里不要求"目标当前仍与镜像不同"——
// "写确实发生"由发起方在写入后的 drift 校验证明,而不是由重放入口猜。
@@ -1020,6 +1149,16 @@ func (t *ProductionLibraryTarget) RestoreDrill(ctx context.Context, backup libra
return fmt.Errorf("演练恢复阶段 %s 失败: %w", stage, err)
}
}
+ // canary 是这次演练唯一新建的行:回放后必须精确消失(不是"整库看起来差不多")。
+ for _, id := range backup.WriteSet.CreatedIDs("knowledge_tags") {
+ var exists bool
+ if err := t.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)`, id).Scan(&exists); err != nil {
+ return fmt.Errorf("读取演练 canary 残留失败: %w", err)
+ }
+ if exists {
+ return fmt.Errorf("演练恢复后 canary %s 仍在目标中", id)
+ }
+ }
after, err := universeFingerprint(ctx, t.pool)
if err != nil {
return fmt.Errorf("演练恢复后读取整库指纹失败: %w", err)
@@ -1045,6 +1184,9 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel
if err := backup.WriteSet.Verify(); err != nil {
return fmt.Errorf("写集合记录校验失败,拒绝恢复:%w", err)
}
+ if backup.WriteSet.IsRestoreDrill() {
+ return errors.New("演练恢复计划不能走发布恢复入口:请使用同作业的恢复演练路径")
+ }
// 身份合同与演练共用同一份判定(libraryrelease.WriteSet.ValidateRestoreBinding):
// 来源快照 ≠ 写前镜像、同一预备份、同一 tenant/KB,缺一不可。
if err := backup.WriteSet.ValidateRestoreBinding(backup.SnapshotID, t.scope.TenantID, t.scope.KnowledgeBaseID); err != nil {
diff --git a/scripts/test-scope.json b/scripts/test-scope.json
index b6c8a40..d64bdf1 100644
--- a/scripts/test-scope.json
+++ b/scripts/test-scope.json
@@ -430,6 +430,7 @@
"reason": "真实隔离PG/ES/MinIO;验证mapping哈希不变和成功后无暂态marker,默认CI无这些服务"
},
{"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestCandidateEmptyTargetFixture","reason":"需显式一次性本机PG/ES/MinIO与运行器注入的隔离库/归档根:为候选页面装配并封存独立 fullshape 测试来源、证空目标、登记 test_sealed;普通CI跳过不是通过"},
- {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestCandidateAttestation","reason":"需运行器注入一次性 ed25519 种子与四份真实证据摘要:续签 object-gateway 审计(有效期上限1h);普通CI没有注入,跳过不是通过"}
+ {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestCandidateAttestation","reason":"需运行器注入一次性 ed25519 种子与四份真实证据摘要:续签 object-gateway 审计(有效期上限1h);普通CI没有注入,跳过不是通过"},
+ {"package":"knowledge-governance/cmd/worker","test":"TestThreeBlockersVerifyChain","reason":"需外置 runner 创建的一次性隔离 PG/ES/MinIO 加真实入口/网关容器与 Automatic+DS01 授权:完整 verify 与同 job recover 纵向链;普通CI没有这些资源,跳过不是通过"}
]
}
diff --git a/web/src/RestoreVerificationSection.mounted.test.tsx b/web/src/RestoreVerificationSection.mounted.test.tsx
index 7cbf9e2..c749835 100644
--- a/web/src/RestoreVerificationSection.mounted.test.tsx
+++ b/web/src/RestoreVerificationSection.mounted.test.tsx
@@ -158,6 +158,47 @@ describe('RestoreVerificationSection mounted (offline fixtures)', () => {
expect(onComplete).toHaveBeenCalledWith('libverify-internal-1')
})
+ it('切 scope 后迟到的旧 POST 不能污染新目标或触发父级旧作业回调', async () => {
+ const onAccepted = vi.fn()
+ const onComplete = vi.fn()
+ let resolvePost: ((value: unknown) => void) | null = null
+ vi.stubGlobal('fetch', async (url: string, init?: { method?: string }) => {
+ if (init?.method === 'POST') {
+ return new Promise((resolve) => {
+ resolvePost = resolve
+ })
+ }
+ return jsonResponse(String(url).includes('scope=external') ? externalView : internalView)
+ })
+ await act(async () => {
+ root.render(
+
+ )
+ })
+ // 旧 internal POST 挂起时切到 external:external 的真实观测已经完成。
+ await act(async () => {
+ ;[...container.querySelectorAll('button')].find((item) => item.textContent?.includes('跑一次恢复验证'))!.click()
+ })
+ await act(async () => {
+ root.render(
+
+ )
+ })
+ expect(container.textContent).toContain('fixture-external')
+ expect(container.textContent).not.toContain('已验证(当前目标)')
+ expect(container.textContent).not.toContain('old-internal-job')
+ // 放行旧 internal POST:它的 then/onAccepted/reload 必须全部失效。
+ await act(async () => {
+ resolvePost?.(jsonResponse({ job: { id: 'verify-internal-new', scope: 'internal' }, reused: false }, 202))
+ })
+ expect(container.textContent).toContain('fixture-external')
+ expect(container.textContent).not.toContain('已验证(当前目标)')
+ expect(container.textContent).not.toContain('old-internal-job')
+ expect(container.textContent).not.toContain('verify-internal-new')
+ expect(onAccepted).not.toHaveBeenCalled()
+ expect(onComplete).not.toHaveBeenCalled()
+ })
+
it('HTTP 失败按失败显示,不把响应当通过证据', async () => {
vi.stubGlobal('fetch', async () =>
jsonResponse({ scope: 'internal', verified: true, reason: '证据读取失败', target, history: [] }, 503)
diff --git a/web/src/RestoreVerificationSection.tsx b/web/src/RestoreVerificationSection.tsx
index 5055711..c53d51f 100644
--- a/web/src/RestoreVerificationSection.tsx
+++ b/web/src/RestoreVerificationSection.tsx
@@ -112,8 +112,15 @@ export function RestoreVerificationSection({
const mounted = useRef(true)
const generation = useRef(0)
const viewScopeRef = useRef('')
+ // scopeRef 是"当前渲染的 scope":异步回调必须同时核对 scope 与世代,
+ // 只比 GET generation 会被迟到的旧 POST 重新递增而失效(审核 B5-R1)。
+ const scopeRef = useRef(scope)
+ const scopeEpoch = useRef(0)
+ const actionSeq = useRef(0)
const acceptedJob = useRef('')
const sawRunning = useRef(false)
+ const onAcceptedRef = useRef(onAccepted)
+ onAcceptedRef.current = onAccepted
const onCompleteRef = useRef(onComplete)
onCompleteRef.current = onComplete
@@ -122,7 +129,7 @@ export function RestoreVerificationSection({
return fetchLibraryVerification(targetScope, signal)
.then((next) => {
// 晚到的旧 scope/旧代次响应一律丢弃:它不能改写当前 scope 的观测。
- if (!mounted.current || request !== generation.current) return
+ if (!mounted.current || request !== generation.current || targetScope !== scopeRef.current) return
setView(next)
viewScopeRef.current = next.scope
setStale(false)
@@ -141,7 +148,7 @@ export function RestoreVerificationSection({
}
})
.catch((err: unknown) => {
- if (!mounted.current || request !== generation.current) return
+ if (!mounted.current || request !== generation.current || targetScope !== scopeRef.current) return
if ((err as Error)?.name === 'AbortError') return
// 刷新失败:同 scope 的旧结果保留但明确标成过期;别的 scope 的旧结果直接丢掉。
setStale(true)
@@ -152,6 +159,11 @@ export function RestoreVerificationSection({
useEffect(() => {
mounted.current = true
+ scopeRef.current = scope
+ // 切 scope 使所有在飞动作失效:旧 scope 的 POST/回调不能写回新目标。
+ scopeEpoch.current += 1
+ actionSeq.current = 0
+ setPending(false)
// 切 scope:旧 scope 的证据不能继续显示为"当前目标已验证"。
setView(null)
viewScopeRef.current = ''
@@ -173,21 +185,37 @@ export function RestoreVerificationSection({
}, [scope])
const onRun = useCallback(() => {
+ const actionScope = scope
+ const actionEpoch = scopeEpoch.current
+ const action = ++actionSeq.current
+ // 动作仍然有效 = 组件挂着 + 还是同一个 scope 世代 + 仍是最新一次点击。
+ const current = () =>
+ mounted.current &&
+ scopeEpoch.current === actionEpoch &&
+ scopeRef.current === actionScope &&
+ actionSeq.current === action
setPending(true)
setError(null)
setNotice('')
- enqueueLibraryVerification(scope)
+ enqueueLibraryVerification(actionScope)
.then((accepted) => {
+ if (!current() || accepted.scope !== actionScope) return
acceptedJob.current = accepted.job_id
sawRunning.current = false
setNotice(`已投递恢复验证作业 ${accepted.job_id};页面会自动刷新进度与终态,父级作业/维护/闸口同步更新。`)
// 父页面必须立刻知道"有一个验证在跑":否则它不会轮询 overview/维护。
- onAccepted?.(accepted.job_id)
- void reload(scope)
+ onAcceptedRef.current?.(accepted.job_id)
+ void reload(actionScope)
})
- .catch((err: unknown) => setError(err instanceof Error ? err.message : '投递恢复验证失败'))
- .finally(() => setPending(false))
- }, [scope, reload, onAccepted])
+ .catch((err: unknown) => {
+ if (!current()) return
+ setError(err instanceof Error ? err.message : '投递恢复验证失败')
+ })
+ .finally(() => {
+ // 旧动作的 finally 不能清掉新 scope 新动作的 pending。
+ if (current()) setPending(false)
+ })
+ }, [scope, reload])
return (