From b4add6c9b0912294ceb9e00a8d54a4bd955f9135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Tue, 3 Nov 2026 13:02:42 +0800 Subject: [PATCH] =?UTF-8?q?fix(library):=20=E6=94=B6=E5=8F=A3=20B1-R1/B2-R?= =?UTF-8?q?1/B5-R1=20=E5=B9=B6=E8=AE=A9=E6=81=A2=E5=A4=8D=E9=93=BE?= =?UTF-8?q?=E7=BB=8F=E8=BF=87=E6=AD=A3=E5=BC=8F=E9=98=9F=E5=88=97=E7=BB=88?= =?UTF-8?q?=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B1-R1: 演练写集改为受限 restore-drill(V2 信封/contract 1/schema 摘要,零对象、零索引、零本地映射,来源身份必填)。B2-R1: verify 作业先建不可变 operation 并携带 operation_id,同 job recover 的网关 claim 同时携带原 operation 与 recovery 身份。B5-R1: 前端按 scope 世代与动作序号丢弃迟到响应,不污染新 scope 或父级旧回调。harness 的 recovery 改由生产 jobs.Worker.RunOnce 消费并读回 jobs 终态;原失败窗口作业保持 failed,不改写历史。 --- CHANGELOG.md | 15 + .../library_three_blockers_chain_test.go | 1202 +++++++++++++++++ internal/httpapi/library_verify.go | 18 +- internal/httpapi/library_verify_test.go | 8 + internal/libraryrelease/writeset.go | 29 + internal/libraryverify/canary_test.go | 8 +- internal/libraryverify/verify.go | 11 + .../v071/gateway_object_store.go | 6 + .../v071/library_production_target.go | 148 +- scripts/test-scope.json | 3 +- ...estoreVerificationSection.mounted.test.tsx | 41 + web/src/RestoreVerificationSection.tsx | 44 +- 12 files changed, 1515 insertions(+), 18 deletions(-) create mode 100644 cmd/worker/library_three_blockers_chain_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fc36ce4..eb4a075 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 开发快照(B1-R1 / B2-R1 / B5-R1 真实链收口) + +### Fixed + +- **B1-R1 受限演练恢复计划**:写集合新增 `restore-drill` 种类——共享 V2 信封、contract 1 与 schema 摘要, + 对象/索引/本地 ID 映射必须为空,来源 tenant/KB 必填;任一项非空即拒绝,旧执行器不能把它当发布计划 + 接管。恢复演练入口 `RestoreDrill` 与对象网关签发的零对象计划使用同一 attempt/写前镜像身份。 +- **B2-R1 verify 作业的不可变 operation**:`POST /api/v1/library/verifications` 在创建作业前先建 + operation(`ensureLibraryIntent`),作业携带 `operation_id` 与稳定幂等键;同一幂等键对应不同请求返回 + 409。worker owned attempt 与同 job recover 的网关 claim 都按原 operation + recovery job/operation + 核验,缺 operation 的 verify 作业在真实恢复中 fail closed。 +- **B5-R1 迟到响应不得污染新 scope**:恢复验证面板按 scope 世代 + 动作序号判定在飞动作;切 scope 即作废 + 旧 POST/回调,迟到响应不能改写新 scope 观测、不能触发父级旧 job 回调、不能清掉新动作的 pending。 + 新增真实挂载反例用例覆盖旧 internal POST 晚到的场景。 + ## 开发快照(GPT-6 正常 UI 发布审核 B1–B5 修复) ### Fixed diff --git a/cmd/worker/library_three_blockers_chain_test.go b/cmd/worker/library_three_blockers_chain_test.go new file mode 100644 index 0000000..e0e22e0 --- /dev/null +++ b/cmd/worker/library_three_blockers_chain_test.go @@ -0,0 +1,1202 @@ +package main + +// 开发隔离集成:B1-R1 / B2-R1 的真实纵向链。 +// +// 只由外置 runner 在一次性资源上启动(KG_THREE_RUN 非空)。它使用真实 +// PostgreSQL / Elasticsearch / MinIO、真实对象网关与入口容器,走正式共享装配 +// (librarystack.OpenBaselineStack / OpenProductionStack,Automatic 网关保留)。 +// 链路: +// API POST /library/verifications +// -> 治理 operation/job(B2) +// -> 真实 owned attempt(worker claim) +// -> 两次真实 Prepare + 受限演练计划/ledger 写前登记 +// -> canary 漂移 +// -> 四阶段真实恢复(database/wiki/resources/index) +// -> 内容核验 + 真实入口恢复/维护释放(B1) +// 失败后:API POST /library/maintenance/recover -> 真实 owned attempt +// -> 同一 job/backup/原 operation + 非 nil DS01 gate -> RestoreDrill(B2) +// -> 网关按 recovery claim 校验零对象计划。 +// +// 这不是 8088 页面验收,也不是生产执行;所有资源由 runner 创建前登记、清理按 +// 不可变 ID/ownership/readback。 + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "golang.org/x/crypto/bcrypt" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/objectgateway" + postgresstore "knowledge-governance/internal/platform/postgres" + v071 "knowledge-governance/internal/releaseadapter/v071" +) + +const tbTargetTenant = 31001 +const tbTargetKB = "kb-verify-chain" +const tbSentinelTenant = 31999 +const tbSentinelKB = "kb-sentinel" + +const tbSchema = ` +CREATE TABLE schema_migrations(version int, dirty bool); +INSERT INTO schema_migrations VALUES(74,false); +CREATE TABLE knowledge_bases(id text PRIMARY KEY, tenant_id bigint, indexing_strategy jsonb, deleted_at timestamptz, + embedding_model_id text, vector_store_id text); +CREATE TABLE models(id text PRIMARY KEY, name text, parameters jsonb, deleted_at timestamptz); +CREATE TABLE vector_stores(id text PRIMARY KEY, engine_type text, index_config jsonb, deleted_at timestamptz); +CREATE TABLE knowledges(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, parse_status text, + pending_subtasks_count int, file_path text, deleted_at timestamptz); +CREATE TABLE chunks(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, knowledge_id text, + content text, seq_id bigint, deleted_at timestamptz); +CREATE TABLE knowledge_tags(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, name text, deleted_at timestamptz); +CREATE TABLE knowledge_tag_relations(knowledge_id text, tag_id text, tenant_id bigint); +CREATE TABLE resources(id text PRIMARY KEY, handle text, tenant_id bigint, physical_path text, provider text, + size bigint, storage_backend_id text, deleted_at timestamptz); +CREATE TABLE resource_bindings(id text PRIMARY KEY, tenant_id bigint, resource_id text, owner_type text, + owner_id text, relation text); +CREATE TABLE wiki_folders(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, deleted_at timestamptz); +CREATE TABLE wiki_pages(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, slug text, content text, + status text, deleted_at timestamptz); +CREATE TABLE wiki_page_issues(id text PRIMARY KEY, tenant_id bigint, knowledge_base_id text, deleted_at timestamptz); +CREATE TABLE wiki_log_entries(id bigint PRIMARY KEY, tenant_id bigint, knowledge_base_id text); +CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz); +CREATE TABLE task_pending_ops(tenant_id bigint, task_type text, scope text, scope_id text); +` + +type tbEnv struct { + root string + pg *pgxpool.Pool + gov *pgxpool.Pool + store *postgresstore.Store + config appconfig.Config + api string + ingress string + gatewayAddr string + bin string + project string + index string + bucket string + s3Endpoint string + s3Access string + s3Secret string + readerKey string + readerSec string + archiveRoot string + workerID string +} + +func tbGetenv(t *testing.T, name string) string { + t.Helper() + value := os.Getenv(name) + if value == "" { + t.Fatalf("missing required isolated env %s", name) + } + return value +} + +func tbWriteJSON(t *testing.T, root, name string, value any) { + t.Helper() + raw, err := json.MarshalIndent(value, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, name), append(raw, '\n'), 0600); err != nil { + t.Fatal(err) + } +} + +func tbStart(t *testing.T, root, label, binary string, env map[string]string) *exec.Cmd { + t.Helper() + cmd := exec.Command(binary) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} + for key, value := range env { + cmd.Env = append(cmd.Env, key+"="+value) + } + log, err := os.OpenFile(filepath.Join(root, label+".log"), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) + if err != nil { + t.Fatal(err) + } + cmd.Stdout, cmd.Stderr = log, log + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + log.Close() + }) + return cmd +} + +func tbS3Locator(tenant uint64, id string) string { + return fmt.Sprintf("storage://backend-target/minio://%s/%d/%s/a.png", tbGetenvID("KG_THREE_BUCKET"), tenant, id) +} + +// tbWaitHTTP 等待前台服务真正绑定端口后再发业务请求;失败时把服务日志带进断言。 +func tbWaitHTTP(t *testing.T, url, logPath string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + resp, err := http.Get(url) + if err == nil { + resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 500 { + return + } + } + if time.Now().After(deadline) { + raw, _ := os.ReadFile(logPath) + t.Fatalf("service never became ready: %s (%v); log: %s", url, err, strings.TrimSpace(string(raw))) + } + time.Sleep(100 * time.Millisecond) + } +} + +func tbGetenvID(name string) string { + if v := os.Getenv(name); v != "" { + return v + } + return "fixture-bucket" +} + +func TestThreeBlockersVerifyChain(t *testing.T) { + root := os.Getenv("KG_THREE_RUN") + if root == "" { + t.Skip("requires the dedicated isolated three-blockers runner (KG_THREE_RUN)") + } + if os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" { + t.Fatal("isolated three-blockers chain requires KG_LIBRARY_TEST_ISOLATED=1") + } + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute) + t.Cleanup(cancel) + + env := &tbEnv{ + root: root, api: tbGetenv(t, "KG_THREE_API_ADDR"), ingress: tbGetenv(t, "KG_THREE_INGRESS_URL"), + gatewayAddr: tbGetenv(t, "KG_THREE_GATEWAY_ADDR"), bin: tbGetenv(t, "KG_THREE_BIN"), + project: tbGetenv(t, "KG_THREE_PROJECT"), index: tbGetenv(t, "KG_THREE_INDEX"), + bucket: tbGetenv(t, "KG_THREE_BUCKET"), s3Endpoint: tbGetenv(t, "KG_LIBRARY_TEST_S3_ENDPOINT"), + s3Access: tbGetenv(t, "KG_LIBRARY_TEST_S3_ACCESS_KEY"), s3Secret: tbGetenv(t, "KG_LIBRARY_TEST_S3_SECRET_KEY"), + readerKey: tbGetenv(t, "KG_THREE_READER_KEY"), readerSec: tbGetenv(t, "KG_THREE_READER_SECRET"), + archiveRoot: filepath.Join(root, "archive"), workerID: tbGetenv(t, "KG_THREE_WORKER_ID"), + } + if err := os.MkdirAll(env.archiveRoot, 0700); err != nil { + t.Fatal(err) + } + + // ---------- 1) 业务库 / 目标现场 ---------- + pool, err := pgxpool.New(ctx, tbGetenv(t, "KG_LIBRARY_TEST_DATABASE_URL")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + env.pg = pool + if _, err := pool.Exec(ctx, tbSchema); err != nil { + t.Fatalf("business schema: %v", err) + } + tbSeedTarget(t, ctx, env) + + // ---------- 2) 治理库 / 网关 / API ---------- + gov, err := pgxpool.New(ctx, tbGetenv(t, "KG_THREE_GOVERNANCE_DSN")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(gov.Close) + env.gov = gov + if err := postgresstore.ApplyMigrations(ctx, gov); err != nil { + t.Fatalf("governance migrations: %v", err) + } + if _, err := gov.Exec(ctx, objectgateway.Schema); err != nil { + t.Fatalf("gateway schema: %v", err) + } + // 业务/API/worker/网关必须共享同一个 target 身份:API 建库作业时写的是 + // appconfig.LibraryTargetIdentity(),worker 的 LibraryJob 查询按它过滤。 + workerEnv := tbConfigEnv(env) + config, err := appconfig.Load(func(key string) string { return workerEnv[key] }) + if err != nil { + t.Fatalf("app config: %v", err) + } + env.config = config + targetID := config.LibraryTargetIdentity() + env.store = postgresstore.NewStore(gov).WithLibraryTarget(targetID) + readerDSN, readerRole := tbCreateReaderRole(t, ctx, gov, tbGetenv(t, "KG_THREE_GOVERNANCE_DSN")) + tbProbeReaderPrivileges(t, ctx, root, readerDSN, readerRole) + tbStartGateway(t, ctx, env, readerDSN, targetID) + + // 入口/写者容器由 runner 建好;先证明拓扑与原状态可读。 + manager, err := openLibraryMaintenance(config) + if err != nil { + t.Fatalf("maintenance wiring: %v", err) + } + if err := manager.VerifyTopology(ctx); err != nil { + t.Fatalf("topology: %v", err) + } + manager.Close() + + apiEnv := map[string]string{} + for key, value := range workerEnv { + apiEnv[key] = value + } + apiEnv["APP_ADDR"] = env.api + apiEnv["GOVERNANCE_SPOOL_ROOT"] = filepath.Join(root, "spool") + apiEnv["RELEASE_WORKER_ENABLED"] = "false" + if err := os.MkdirAll(apiEnv["GOVERNANCE_SPOOL_ROOT"], 0700); err != nil { + t.Fatal(err) + } + tbStart(t, root, "api", filepath.Join(env.bin, "api"), apiEnv) + tbWaitHTTP(t, "http://"+env.api+"/health", filepath.Join(root, "api.log"), 60*time.Second) + api := tbLogin(t, ctx, gov, "http://"+env.api) + + // ---------- 3) 正常完整验证链路 ---------- + beforePG := tbDump(t, ctx, pool) + beforeIndex := tbIndexReadback(t, ctx, env) + tbWriteJSON(t, root, "10-before-pg.json", beforePG) + tbWriteJSON(t, root, "11-before-index.json", beforeIndex) + + job1 := tbEnqueueVerify(t, api) + tbWriteJSON(t, root, "12-verify-job.json", job1) + if job1.OperationID == "" { + t.Fatal("API-created verify job carries no operation_id (B2-R1)") + } + queue1 := tbClaimQueue(t, ctx, env.store, "library.verify") + if err := runLibraryVerify(ctx, env.store, config, *queue1); err != nil { + t.Fatalf("normal verify failed: %v", err) + } + after1, err := env.store.LibraryJob(ctx, job1.ID) + if err != nil { + t.Fatal(err) + } + tbWriteJSON(t, root, "13-verify-job-after.json", after1) + if after1.State != governance.LibraryJobSucceeded { + t.Fatalf("normal verify job state=%s error=%s", after1.State, after1.Error) + } + evidence, found, err := env.store.LatestLibraryRestoreVerification(ctx, "internal") + if err != nil || !found { + t.Fatalf("verification evidence missing: found=%v err=%v", found, err) + } + tbWriteJSON(t, root, "14-verification-evidence.json", evidence) + if evidence.Status != governance.LibraryVerificationPassed || evidence.JobID != job1.ID { + t.Fatalf("verification evidence=%#v", evidence) + } + // 内容与 sentinel:恢复后必须逐行/逐文档回到写前镜像,且无关租户不变。 + if diff := tbDiffDump(beforePG, tbDump(t, ctx, pool)); diff != "" { + t.Fatalf("restored business content differs: %s", diff) + } + if diff := tbDiffIndex(beforeIndex, tbIndexReadback(t, ctx, env)); diff != "" { + t.Fatalf("restored index differs: %s", diff) + } + tbAssertMaintenanceReleased(t, ctx, env) + tbAssertIngressHealthy(t, env) + // 演练计划:受限 V2、零对象/索引/本地映射,且已登记为 production_prebackup。 + ledger := tbFindDrillLedger(t, ctx, env, job1.ID) + tbWriteJSON(t, root, "15-drill-plan.json", map[string]any{ + "digest": ledger.Digest, "job": ledger.JobID, "kind": ledger.Kind, "version": ledger.Version, + "contract_version": ledger.ContractVersion, "objects": len(ledger.Objects), "index": len(ledger.Index), + "local_ids": len(ledger.LocalIDs), "backup_snapshot": ledger.BackupSnapshotID, + "target_snapshot": ledger.TargetSnapshotID, "rows": ledger.Rows, + "schema_digest": ledger.SchemaDigest, + }) + if ledger.Kind != libraryrelease.WriteSetKindRestoreDrill || ledger.Version != libraryrelease.MigrationWriteSetVersion || + len(ledger.Objects) != 0 || len(ledger.Index) != 0 || len(ledger.LocalIDs) != 0 { + t.Fatalf("drill plan is not restricted: %#v", ledger) + } + // 对象网关确实签发了零对象计划(controller delivery 行),不是跳过网关。 + var deliveries int + if err := gov.QueryRow(ctx, `SELECT count(*) FROM kg_object_gateway_deliveries WHERE job=$1`, job1.ID).Scan(&deliveries); err != nil { + t.Fatal(err) + } + if deliveries != 1 { + t.Fatalf("object gateway deliveries for verify job = %d, want 1", deliveries) + } + + // ---------- 4) 失败后同一 job 恢复链路(DS01 gate 非 nil) ---------- + proxy := tbNewRefusingProxy(t, env, tbGetenv(t, "KG_LIBRARY_TEST_ELASTIC_URL")) + // 让 worker 装配指向代理:其余字段不变。 + workerEnv["LIBRARY_PRODUCTION_ELASTIC_URL"] = proxy.url + config2, err := appconfig.Load(func(key string) string { return workerEnv[key] }) + if err != nil { + t.Fatal(err) + } + config = config2 + + job2 := tbEnqueueVerify(t, api) + tbWriteJSON(t, root, "20-recover-job.json", job2) + queue2 := tbClaimQueue(t, ctx, env.store, "library.verify") + done := make(chan error, 1) + go func() { done <- runLibraryVerify(ctx, env.store, config, *queue2) }() + tbWaitStage(t, ctx, env, job2.ID, "restore", 10*time.Minute) + proxy.RefuseNextBulk() + runErr := <-done + if runErr == nil { + t.Fatal("verify with injected ES refusal after canary unexpectedly succeeded") + } + after2, err := env.store.LibraryJob(ctx, job2.ID) + if err != nil { + t.Fatal(err) + } + tbWriteJSON(t, root, "21-recover-job-failed.json", after2) + tbWriteJSON(t, root, "22-recover-run-error.json", map[string]string{"error": runErr.Error()}) + if after2.State != governance.LibraryJobFailed { + t.Fatalf("failed verify state=%s", after2.State) + } + inst, err := env.store.InstanceMaintenanceOf(ctx, env.project) + if err != nil || inst == nil || inst.ReleasedAt != nil || inst.State != "needs_restore" || inst.IngressState != "closed" { + t.Fatalf("held instance maintenance = %#v err=%v", inst, err) + } + if err := tbIngressClosed(env); err != nil { + t.Fatalf("ingress must be closed during held window: %v", err) + } + // 真实 UI/API 恢复入口:同一个原窗口 job。 + recoveryQueue := tbPostRecover(t, api, job2.ID) + tbWriteJSON(t, root, "23-recover-queue.json", recoveryQueue) + proxy.Disarm() + // 真实消费路径:与生产 worker 同一个 jobs.Worker 外壳 + // (claim -> handler -> heartbeat -> Complete/Fail)。 + // 直接调用 handler 只会执行恢复函数,不会写回 jobs 行的正式终态,页面/队列 + // 会永远看到 running;这里必须让同一外壳完成并按 id 读回终态。 + recoveryShell, err := jobs.NewWorker(jobs.WorkerConfig{ + Queue: env.store, + WorkerID: config.WorkerID, + LeaseDuration: config.LeaseDuration, + RetryDelay: config.RetryDelay, + Handlers: map[string]jobs.Handler{ + libraryRecoverJobType: admittanceGuarded(nil, func(ctx context.Context, job jobs.Job) error { + return runLibraryRecovery(ctx, env.store, config, job) + }), + }, + }) + if err != nil { + t.Fatalf("recovery worker shell: %v", err) + } + claimed, err := recoveryShell.RunOnce(ctx) + if err != nil { + t.Fatalf("same-job recovery through queue shell failed: %v", err) + } + if !claimed { + t.Fatalf("recovery queue %s was not claimed by the worker shell", recoveryQueue.ID) + } + queueFinal := tbWaitQueueTerminal(t, ctx, env.store, recoveryQueue.ID) + tbWriteJSON(t, root, "24-recover-complete.json", map[string]any{"window_job": job2.ID, "recovery_queue": recoveryQueue.ID, "terminal": queueFinal}) + if queueFinal.State != jobs.StateCompleted || queueFinal.LeaseOwner != "" || queueFinal.LastError != "" { + t.Fatalf("recovery queue terminal state=%s owner=%q last_error=%q, want completed with released lease", + queueFinal.State, queueFinal.LeaseOwner, queueFinal.LastError) + } + if diff := tbDiffDump(beforePG, tbDump(t, ctx, pool)); diff != "" { + t.Fatalf("recovery business content differs: %s", diff) + } + if diff := tbDiffIndex(beforeIndex, tbIndexReadback(t, ctx, env)); diff != "" { + t.Fatalf("recovery index differs: %s", diff) + } + tbAssertMaintenanceReleased(t, ctx, env) + tbAssertIngressHealthy(t, env) + // 网关 delivery 必须记录 recovery 身份(原 attempt + recovery job/operation), + // 证明不是把 gate 置 nil 绕过。 + var claimRaw []byte + if err := gov.QueryRow(ctx, `SELECT claim FROM kg_object_gateway_deliveries WHERE job=$1 ORDER BY attempt_epoch DESC LIMIT 1`, job2.ID).Scan(&claimRaw); err != nil { + t.Fatal(err) + } + var claim objectgateway.Claim + if err := json.Unmarshal(claimRaw, &claim); err != nil { + t.Fatal(err) + } + tbWriteJSON(t, root, "25-recovery-claim.json", claim) + if claim.OperationID != job2.OperationID || claim.RecoveryJobID == "" || claim.RecoveryOperationID == "" { + t.Fatalf("recovery gateway claim does not carry original+recovery identity: %#v", claim) + } + // 终态读回:原失败作业保持 failed(历史语义不改写,不要求它变成功); + // recovery 队列终态由同一消费外壳写回,且已失租。 + finalWindow, err := env.store.LibraryJob(ctx, job2.ID) + if err != nil { + t.Fatal(err) + } + if finalWindow.State != governance.LibraryJobFailed { + t.Fatalf("original failed window job was rewritten to %s (must stay failed)", finalWindow.State) + } + finalInst, err := env.store.InstanceMaintenanceOf(ctx, env.project) + if err != nil || finalInst == nil || finalInst.ReleasedAt == nil || + finalInst.State != "released" || finalInst.IngressState != "open" { + t.Fatalf("instance maintenance final = %#v err=%v", finalInst, err) + } + var scopeState, scopeIngress string + var scopeReleased *time.Time + if err := gov.QueryRow(ctx, `SELECT state,ingress_state,released_at FROM library_maintenance WHERE scope='internal'`). + Scan(&scopeState, &scopeIngress, &scopeReleased); err != nil { + t.Fatal(err) + } + if scopeReleased == nil || scopeState != "verified" || scopeIngress != "open" { + t.Fatalf("scope maintenance final state=%s ingress=%s released=%v", scopeState, scopeIngress, scopeReleased) + } + sentinelPG := tbUnrelatedSentinel(t, ctx, pool) + if sentinelPG != "sentinel-doc/sentinel-chunk" { + t.Fatalf("unrelated PG sentinel changed: %s", sentinelPG) + } + sentinelIndex := "missing" + if _, ok := tbIndexReadback(t, ctx, env)["sentinel-chunk"]; ok { + sentinelIndex = "sentinel-chunk" + } + if sentinelIndex != "sentinel-chunk" { + t.Fatal("unrelated index sentinel missing after recovery") + } + tbWriteJSON(t, root, "26-final-state.json", map[string]any{ + "window_job": job2.ID, "window_state": string(finalWindow.State), + "recovery_queue": recoveryQueue.ID, "recovery_queue_state": string(queueFinal.State), + "recovery_queue_lease_owner": queueFinal.LeaseOwner, + "consumer": "jobs.Worker.RunOnce (claim -> handler -> heartbeat -> Complete/Fail)", + "maintenance": map[string]any{ + "instance_state": finalInst.State, "instance_ingress": finalInst.IngressState, + "instance_released_at": finalInst.ReleasedAt, + "scope_state": scopeState, "scope_ingress": scopeIngress, "scope_released_at": scopeReleased, + }, + "ingress_healthy": true, "content_identical": true, "index_identical": true, + "unrelated_sentinel": map[string]string{"pg": sentinelPG, "index": sentinelIndex}, + "claim": claim, + }) +} + +// tbSeedTarget 造一个有内容的隔离目标:文档/分块/资源/绑定/Wiki + 一个无关租户 sentinel。 +func tbSeedTarget(t *testing.T, ctx context.Context, env *tbEnv) { + t.Helper() + locator := func(id string) string { + return fmt.Sprintf("storage://backend-target/minio://%s/%d/%s/a.png", env.bucket, tbTargetTenant, id) + } + // 对象字节由 runner 用 admin 凭据写入(worker 只有只读身份)。 + statements := []string{ + `INSERT INTO knowledge_bases VALUES('` + tbTargetKB + `',31001,'{}',NULL,'fixture-model','fixture-vector')`, + `INSERT INTO models VALUES('fixture-model','fixture-embed','{"embedding_parameters":{"dimension":1024}}',NULL)`, + `INSERT INTO vector_stores VALUES('fixture-vector','elasticsearch_v8','{}',NULL)`, + `INSERT INTO storage_backends VALUES('backend-target',31001,'minio','{"bucket_name":"` + env.bucket + `"}',now())`, + `INSERT INTO knowledges VALUES('doc-target',31001,'` + tbTargetKB + `','completed',0,'',NULL)`, + `INSERT INTO chunks VALUES('chunk-target',31001,'` + tbTargetKB + `','doc-target','图:resource://aaaaaaaaaaaaaaaaaaaaaa',10,NULL)`, + `INSERT INTO knowledge_tags VALUES('tag-target',31001,'` + tbTargetKB + `','目标标签',NULL)`, + `INSERT INTO resources VALUES('res-target','aaaaaaaaaaaaaaaaaaaaaa',31001,'` + locator("res-target") + `','minio',8,'backend-target',NULL)`, + `INSERT INTO resource_bindings VALUES('bind-target',31001,'res-target','knowledge','doc-target','attachment')`, + `INSERT INTO wiki_pages(id,tenant_id,knowledge_base_id,slug,content,status) VALUES('wiki-target',31001,'` + tbTargetKB + `','index','目标 wiki','published')`, + `INSERT INTO wiki_folders VALUES('folder-target',31001,'` + tbTargetKB + `',NULL)`, + `INSERT INTO wiki_page_issues VALUES('issue-target',31001,'` + tbTargetKB + `',NULL)`, + `INSERT INTO wiki_log_entries VALUES(1,31001,'` + tbTargetKB + `')`, + // 无关 sentinel:另一个租户/知识库,恢复不允许碰。 + `INSERT INTO knowledges VALUES('sentinel-doc',31999,'` + tbSentinelKB + `','completed',0,'',NULL)`, + `INSERT INTO chunks VALUES('sentinel-chunk',31999,'` + tbSentinelKB + `','sentinel-doc','sentinel 正文',1,NULL)`, + } + for _, statement := range statements { + if _, err := env.pg.Exec(ctx, statement); err != nil { + t.Fatalf("seed target: %v\n%s", err, statement) + } + } + // 索引:目标文档 + sentinel 文档;runner 已创建映射。 + tbIndexDoc(t, ctx, env, "chunk-target", tbTargetTenant, tbTargetKB, "doc-target", "图:aaaaaaaaaaaaaaaaaaaaaa") + tbIndexDoc(t, ctx, env, "sentinel-chunk", tbSentinelTenant, tbSentinelKB, "sentinel-doc", "sentinel 正文") + // 对象:目标资源 + sentinel 对象。 + tbPutObject(t, ctx, env, locator("res-target"), []byte("12345678")) +} + +func tbIndexDoc(t *testing.T, ctx context.Context, env *tbEnv, id string, tenant uint64, kb, knowledge, content string) { + t.Helper() + vector := strings.TrimSuffix(strings.Repeat("0.1,", 1024), ",") + body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":[%s]}`, + tenant, kb, knowledge, id, id, content, vector) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, + fmt.Sprintf("%s/%s/_doc/%s?refresh=true", tbGetenv(t, "KG_LIBRARY_TEST_ELASTIC_URL"), env.index, id), strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("index doc: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 && resp.StatusCode != 201 { + raw, _ := io.ReadAll(resp.Body) + t.Fatalf("index doc %s => %d %s", id, resp.StatusCode, raw) + } +} + +func tbPutObject(t *testing.T, ctx context.Context, env *tbEnv, locator string, body []byte) { + t.Helper() + store, err := v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{Provider: "minio", Endpoint: env.s3Endpoint, + Region: "us-east-1", AccessKeyID: env.s3Access, SecretAccessKey: env.s3Secret, Bucket: env.bucket, + ForcePathStyle: true, MaxObjectBytes: 64 << 20}) + if err != nil { + t.Fatal(err) + } + if err := store.EnsureBucket(ctx); err != nil { + t.Fatal(err) + } + if err := store.Put(ctx, "minio", locator, bytes.NewReader(body), int64(len(body))); err != nil { + t.Fatalf("put object %s: %v", locator, err) + } +} + +func tbConfigEnv(env *tbEnv) map[string]string { + return map[string]string{ + "KG_LIBRARY_TEST_ISOLATED": "1", "DATABASE_URL": tbGetenvID("KG_THREE_GOVERNANCE_DSN"), + "WORKER_ID": env.workerID, "LIBRARY_WORKER_ROLE": "release-executor", + "WORKER_POLL_INTERVAL": "200ms", "WORKER_LEASE_DURATION": "60s", + "PRODUCTION_RELEASE_REQUESTS_ENABLED": "true", + "PRODUCTION_RELEASE_AUTHORIZER_REF": env.project, + "PRODUCTION_RELEASE_AUTH_TOKEN_FILE": filepath.Join(env.root, "authorization-token"), + "LIBRARY_ARCHIVE_ROOT": env.archiveRoot, + "LIBRARY_CAPACITY_BUDGET_BYTES": "1073741824", + "LIBRARY_PRODUCTION_DATABASE_URL": tbGetenvID("KG_LIBRARY_TEST_DATABASE_URL"), + "LIBRARY_PRODUCTION_ELASTIC_URL": tbGetenvID("KG_LIBRARY_TEST_ELASTIC_URL"), + "LIBRARY_PRODUCTION_INDEX": env.index, + "LIBRARY_PRODUCTION_TENANT_ID": fmt.Sprint(tbTargetTenant), + "LIBRARY_PRODUCTION_KB_INTERNAL": tbTargetKB, + "LIBRARY_PRODUCTION_S3_PROVIDER": "minio", + "LIBRARY_PRODUCTION_S3_ENDPOINT": env.s3Endpoint, + "LIBRARY_PRODUCTION_S3_BUCKET": env.bucket, + "LIBRARY_PRODUCTION_S3_ACCESS_KEY": env.readerKey, + "LIBRARY_PRODUCTION_S3_SECRET_KEY": env.readerSec, + "LIBRARY_TARGET_INSTANCE": env.project, + "LIBRARY_TARGET_PROJECT": env.project, + "LIBRARY_TARGET_WRITERS": "writer", + "LIBRARY_TARGET_NON_WRITERS": "ingress=entry gate,redis=queue,minio=object backend", + "LIBRARY_TARGET_INGRESS_URL": env.ingress, + "LIBRARY_TARGET_INGRESS_SERVICE": "ingress", + "LIBRARY_TARGET_INGRESS_DRAIN_FILE": filepath.Join(env.root, "gate/drain.json"), + "LIBRARY_TARGET_DOCKER_SOCKET": "/var/run/docker.sock", + "LIBRARY_TARGET_REDIS_ADDR": tbGetenvID("KG_THREE_REDIS_ADDR"), + "LIBRARY_TARGET_REDIS_PASSWORD": tbGetenvID("KG_THREE_REDIS_PASSWORD"), + "LIBRARY_TARGET_REDIS_DB": "0", + "LIBRARY_TARGET_JOURNAL_ROOT": filepath.Join(env.root, "maintenance"), + "LIBRARY_EMBEDDING_MODEL": "fixture-model", + "LIBRARY_EMBEDDING_DIMENSION": "1024", + "LIBRARY_RESTORE_DISPATCH_V2": "1", + "LIBRARY_OBJECT_GATEWAY_URL": "http://" + env.gatewayAddr, + "LIBRARY_OBJECT_GATEWAY_TOKEN": tbGetenvID("KG_THREE_DATA_TOKEN"), + "LIBRARY_OBJECT_GATEWAY_GRANT_DIRECTORY": "", + } +} + +func tbCreateReaderRole(t *testing.T, ctx context.Context, admin *pgxpool.Pool, dsn string) (string, string) { + t.Helper() + random := make([]byte, 24) + if _, err := rand.Read(random); err != nil { + t.Fatal(err) + } + suffix := hex.EncodeToString(random) + role := "kg_three_reader_" + suffix[:12] + if _, err := admin.Exec(ctx, "CREATE ROLE "+pgx.Identifier{role}.Sanitize()+" LOGIN PASSWORD '"+suffix+"'"); err != nil { + t.Fatal("create isolated reader role: ", err) + } + t.Cleanup(func() { + cleanup := context.Background() + _, _ = admin.Exec(cleanup, "DROP OWNED BY "+pgx.Identifier{role}.Sanitize()) + _, _ = admin.Exec(cleanup, "DROP ROLE "+pgx.Identifier{role}.Sanitize()) + }) + var schema string + if err := admin.QueryRow(ctx, "SELECT current_schema()").Scan(&schema); err != nil { + t.Fatal(err) + } + if _, err := admin.Exec(ctx, "GRANT USAGE ON SCHEMA "+pgx.Identifier{schema}.Sanitize()+" TO "+pgx.Identifier{role}.Sanitize()); err != nil { + t.Fatal(err) + } + if _, err := admin.Exec(ctx, "GRANT SELECT ON library_jobs,library_operations,library_attempts,library_versions,jobs TO "+pgx.Identifier{role}.Sanitize()); err != nil { + t.Fatal(err) + } + readerDSN, err := tbReaderDSN(dsn, role, suffix) + if err != nil { + t.Fatalf("build reader dsn: %v", err) + } + return readerDSN, role +} + +// tbReaderDSN 显式从 URL 重建 reader 连接串。 +// +// 不能用 pgxpool.Config.ConnString():pgx v5 保存的是解析时的原始连接串, +// 只改 ConnConfig.User/Password 再取 ConnString 会原样返回 admin DSN, +// 连接身份仍是 admin(上轮 gateway_control_reader_privileges_invalid 的根因)。 +func tbReaderDSN(dsn, user, password string) (string, error) { + parsed, err := url.Parse(dsn) + if err != nil { + return "", err + } + if parsed.Scheme != "postgres" && parsed.Scheme != "postgresql" { + return "", fmt.Errorf("reader dsn requires a postgres URL, got scheme %q", parsed.Scheme) + } + if parsed.Host == "" || strings.Trim(parsed.Path, "/") == "" { + return "", fmt.Errorf("reader dsn requires host and database") + } + parsed.User = url.UserPassword(user, password) + return parsed.String(), nil +} + +// tbProbeReaderPrivileges 在实际连接上硬断言 reader 身份与只读权限; +// 03-reader-privileges.json 只是证据,不通过断言就不启动网关。 +func tbProbeReaderPrivileges(t *testing.T, ctx context.Context, root, dsn, wantUser string) { + t.Helper() + if wantUser == "" { + t.Fatal("reader probe requires an explicit expected user") + } + parsed, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatalf("reader dsn parse: %v", err) + } + if parsed.ConnConfig.User != wantUser { + t.Fatalf("reader dsn identity user=%q, want %q", parsed.ConnConfig.User, wantUser) + } + pool, err := pgxpool.NewWithConfig(ctx, parsed) + if err != nil { + t.Fatalf("reader connect: %v", err) + } + defer pool.Close() + var user string + var super, createdb, createrole bool + if err := pool.QueryRow(ctx, `SELECT current_user, rolsuper, rolcreatedb, rolcreaterole FROM pg_roles WHERE rolname=current_user`).Scan(&user, &super, &createdb, &createrole); err != nil { + t.Fatalf("reader identity: %v", err) + } + privileges := map[string]any{"current_user": user, "rolsuper": super, "rolcreatedb": createdb, "rolcreaterole": createrole} + selectOKByTable := map[string]bool{} + dmlOKByTable := map[string]bool{} + for _, table := range []string{"library_jobs", "library_operations", "library_attempts", "library_versions", "jobs"} { + var selectOK, dmlOK bool + if err := pool.QueryRow(ctx, `SELECT has_table_privilege(current_user,$1,'SELECT'), has_table_privilege(current_user,$1,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')`, table).Scan(&selectOK, &dmlOK); err != nil { + t.Fatalf("reader privilege %s: %v", table, err) + } + selectOKByTable[table], dmlOKByTable[table] = selectOK, dmlOK + privileges[table] = map[string]bool{"select": selectOK, "dml": dmlOK} + } + tbWriteJSON(t, root, "03-reader-privileges.json", privileges) + if user != wantUser { + t.Fatalf("reader connection identity current_user=%q, want %q", user, wantUser) + } + if super || createdb || createrole { + t.Fatalf("reader role must not be super/createdb/createrole: %#v", privileges) + } + for _, table := range []string{"library_jobs", "library_operations", "library_attempts", "library_versions", "jobs"} { + if !selectOKByTable[table] || dmlOKByTable[table] { + t.Fatalf("reader role privilege mismatch on %s: select=%v dml=%v", table, selectOKByTable[table], dmlOKByTable[table]) + } + } +} + +// TestThreeBlockersReaderDSNIdentity 证明显式重建会切到 reader 身份并保留 +// host/database/query,且不会回带 admin 密码。真实链中的 +// 03-reader-privileges.json 断言同一 DSN 的实际会话身份。 +func TestThreeBlockersReaderDSNIdentity(t *testing.T) { + const adminDSN = "postgres://gov:admin-secret@db.internal:5432/kg_gov?sslmode=disable&application_name=three-blockers" + const role = "kg_three_reader_0123456789ab" + const password = "reader-secret" + + readerDSN, err := tbReaderDSN(adminDSN, role, password) + if err != nil { + t.Fatal(err) + } + if strings.Contains(readerDSN, "admin-secret") { + t.Fatalf("reader dsn still carries the admin password") + } + parsed, err := url.Parse(readerDSN) + if err != nil { + t.Fatal(err) + } + if parsed.User.Username() != role { + t.Fatalf("reader dsn user=%q, want %q", parsed.User.Username(), role) + } + if got, _ := parsed.User.Password(); got != password { + t.Fatalf("reader dsn password mismatch") + } + if parsed.Host != "db.internal:5432" || parsed.Path != "/kg_gov" { + t.Fatalf("reader dsn changed host/database: %s", readerDSN) + } + if parsed.Query().Get("sslmode") != "disable" || parsed.Query().Get("application_name") != "three-blockers" { + t.Fatalf("reader dsn dropped query: %s", readerDSN) + } + config, err := pgxpool.ParseConfig(readerDSN) + if err != nil { + t.Fatal(err) + } + if config.ConnConfig.User != role || config.ConnConfig.Password != password || + config.ConnConfig.Host != "db.internal" || config.ConnConfig.Port != 5432 || config.ConnConfig.Database != "kg_gov" { + t.Fatalf("pool config did not adopt reader identity: user=%q host=%q port=%d db=%q", + config.ConnConfig.User, config.ConnConfig.Host, config.ConnConfig.Port, config.ConnConfig.Database) + } + if _, err := tbReaderDSN("host=db.internal user=gov dbname=kg_gov", role, password); err == nil { + t.Fatal("keyword/value dsn must be rejected rather than silently keeping the admin identity") + } +} + +func tbStartGateway(t *testing.T, ctx context.Context, env *tbEnv, readerDSN, targetID string) { + t.Helper() + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + evidence := filepath.Join(env.root, "05-identity.json") + raw := []byte(`{"kind":"isolated-three-blockers","writer":"writer","gateway_only_writer":true}`) + if err := os.WriteFile(evidence, raw, 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(raw) + digest := hex.EncodeToString(sum[:]) + scope := objectgateway.Digest([]byte("minio\x00" + env.s3Endpoint + "\x00" + env.bucket)) + payload, err := json.Marshal(objectgateway.Attestation{Protocol: objectgateway.Protocol, Scope: scope, + GatewayID: env.project, IssuedAt: time.Now().Add(-time.Second), ExpiresAt: time.Now().Add(50 * time.Minute), + WriterInventory: digest, PolicyEvidence: digest, DenialEvidence: digest, DrainEvidence: digest}) + if err != nil { + t.Fatal(err) + } + signed, err := json.Marshal(objectgateway.SignedAttestation{Payload: payload, Signature: ed25519.Sign(priv, payload)}) + if err != nil { + t.Fatal(err) + } + audit := filepath.Join(env.root, "audit.json") + if err := os.WriteFile(audit, signed, 0600); err != nil { + t.Fatal(err) + } + tbStart(t, env.root, "gateway", filepath.Join(env.bin, "object-gateway"), map[string]string{ + "OBJECT_GATEWAY_LISTEN": env.gatewayAddr, "OBJECT_GATEWAY_DATABASE_URL": tbGetenvID("KG_THREE_GOVERNANCE_DSN"), + "OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL": readerDSN, "OBJECT_GATEWAY_ARCHIVE_ROOT": env.archiveRoot, + "OBJECT_GATEWAY_TARGET_ID": targetID, "OBJECT_GATEWAY_PROVIDER": "minio", + "OBJECT_GATEWAY_ENDPOINT": env.s3Endpoint, "OBJECT_GATEWAY_BUCKET": env.bucket, + "OBJECT_GATEWAY_REGION": "us-east-1", "OBJECT_GATEWAY_ACCESS_KEY": env.s3Access, + "OBJECT_GATEWAY_SECRET_KEY": env.s3Secret, "OBJECT_GATEWAY_DATA_TOKEN": tbGetenvID("KG_THREE_DATA_TOKEN"), + "OBJECT_GATEWAY_CONTROL_TOKEN": tbGetenvID("KG_THREE_CONTROL_TOKEN"), "OBJECT_GATEWAY_ID": env.project, + "OBJECT_GATEWAY_AUDIT_FILE": audit, "OBJECT_GATEWAY_AUDIT_PUBLIC_KEY": base64.StdEncoding.EncodeToString(pub), + }) + client, err := objectgateway.NewClient("http://"+env.gatewayAddr, tbGetenvID("KG_THREE_DATA_TOKEN"), scope) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(60 * time.Second) + for { + if err := client.ReadyForAcquisition(ctx); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("object gateway never became ready (see gateway.log)") + } + time.Sleep(200 * time.Millisecond) + } +} + +// tbLogin 走真实 API 登录(一次性治理库里的 admin),保留 cookie。 +func tbLogin(t *testing.T, ctx context.Context, gov *pgxpool.Pool, base string) *tbAPI { + t.Helper() + password := hex.EncodeToString(func() []byte { b := make([]byte, 18); _, _ = rand.Read(b); return b }()) + hash, err := bcrypt.GenerateFromPassword([]byte(password), 10) + if err != nil { + t.Fatal(err) + } + if _, err := gov.Exec(ctx, `UPDATE users SET password_hash=$1, must_change_password=false, failed_attempts=0, + locked_until=NULL, state='active' WHERE username='admin'`, string(hash)); err != nil { + t.Fatal(err) + } + jar, err := cookiejar.New(nil) + if err != nil { + t.Fatal(err) + } + api := &tbAPI{base: base, client: &http.Client{Jar: jar, Timeout: 30 * time.Second}} + status, body := api.do(t, http.MethodPost, "/api/v1/auth/login", map[string]string{"username": "admin", "password": password}) + if status != http.StatusOK { + t.Fatalf("admin login => %d %v", status, body) + } + return api +} + +type tbAPI struct { + base string + client *http.Client +} + +func (a *tbAPI) do(t *testing.T, method, path string, payload any) (int, map[string]any) { + t.Helper() + var reader io.Reader + if payload != nil { + raw, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + reader = bytes.NewReader(raw) + } + req, err := http.NewRequest(method, a.base+path, reader) + if err != nil { + t.Fatal(err) + } + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := a.client.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + var body map[string]any + _ = json.Unmarshal(raw, &body) + return resp.StatusCode, body +} + +func tbEnqueueVerify(t *testing.T, api *tbAPI) governance.LibraryJob { + t.Helper() + status, body := api.do(t, http.MethodPost, "/api/v1/library/verifications", map[string]string{"scope": "internal"}) + if status != http.StatusAccepted { + t.Fatalf("enqueue verify => %d %v", status, body) + } + raw, _ := json.Marshal(body["job"]) + var job governance.LibraryJob + if err := json.Unmarshal(raw, &job); err != nil { + t.Fatalf("decode accepted job: %v (%v)", err, body) + } + if job.ID == "" { + t.Fatalf("accepted job missing id: %v", body) + } + return job +} + +func tbPostRecover(t *testing.T, api *tbAPI, windowJobID string) jobs.Job { + t.Helper() + status, body := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover", + map[string]string{"scope": "internal", "window_job_id": windowJobID}) + if status != http.StatusAccepted { + t.Fatalf("recover enqueue => %d %v", status, body) + } + raw, _ := json.Marshal(body["job"]) + var job jobs.Job + if err := json.Unmarshal(raw, &job); err != nil { + t.Fatalf("decode recovery queue job: %v", err) + } + return job +} + +func tbClaimQueue(t *testing.T, ctx context.Context, store *postgresstore.Store, kind string) *jobs.Job { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + job, err := store.Claim(ctx, tbGetenvID("KG_THREE_WORKER_ID"), []string{kind}, time.Now().UTC(), 2*time.Minute) + if err == nil && job != nil { + return job + } + if time.Now().After(deadline) { + t.Fatalf("claim %s queue job: %v", kind, err) + } + time.Sleep(200 * time.Millisecond) + } +} + +// tbWaitQueueTerminal 等同一消费外壳把 jobs 行写回终态后按 id 读回; +// 只读,不修改队列状态。 +func tbWaitQueueTerminal(t *testing.T, ctx context.Context, store *postgresstore.Store, id string) jobs.Job { + t.Helper() + deadline := time.Now().Add(2 * time.Minute) + var last error + for { + list, err := store.ListByType(ctx, "library.recover", 200) + if err == nil { + for _, job := range list { + if job.ID == id { + if job.State.Terminal() { + return job + } + break + } + } + } else { + last = err + } + if time.Now().After(deadline) { + t.Fatalf("recovery queue %s never reached terminal state: %v", id, last) + } + time.Sleep(100 * time.Millisecond) + } +} + +// tbUnrelatedSentinel 读回无关租户的行身份,证明恢复没有碰不属于本次窗口的数据。 +func tbUnrelatedSentinel(t *testing.T, ctx context.Context, pool *pgxpool.Pool) string { + t.Helper() + ids := func(table string) string { + rows, err := pool.Query(ctx, `SELECT id FROM `+table+` WHERE tenant_id=31999 ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + out = append(out, id) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return strings.Join(out, ",") + } + return ids("knowledges") + "/" + ids("chunks") +} + +func tbWaitStage(t *testing.T, ctx context.Context, env *tbEnv, jobID, stage string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + var state string + err := env.gov.QueryRow(ctx, `SELECT state FROM library_job_stages WHERE job_id=$1 AND stage=$2`, jobID, stage).Scan(&state) + if err == nil && (state == "running" || state == "succeeded") { + return + } + var jobState string + _ = env.gov.QueryRow(ctx, `SELECT state FROM library_jobs WHERE id=$1`, jobID).Scan(&jobState) + if jobState == "failed" || jobState == "succeeded" { + t.Fatalf("job %s reached %s before stage %s", jobID, jobState, stage) + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("job %s never reached stage %s", jobID, stage) +} + +// tbRefusingProxy 只在传输层拒绝指定的 ES 请求(不转发),其余原样转发。 +type tbRefusingProxy struct { + url string + refuse atomic.Bool + refused atomic.Int64 +} + +func tbNewRefusingProxy(t *testing.T, env *tbEnv, upstream string) *tbRefusingProxy { + t.Helper() + p := &tbRefusingProxy{} + server := &http.Server{} + _ = server + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + if p.refuse.Load() && r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/_bulk") { + p.refused.Add(1) + http.Error(w, "injected ES restore refusal before dispatch", http.StatusServiceUnavailable) + return + } + req, err := http.NewRequestWithContext(r.Context(), r.Method, upstream+r.URL.RequestURI(), bytes.NewReader(raw)) + if err != nil { + http.Error(w, "proxy request failed", 500) + return + } + req.Header = r.Header.Clone() + req.Header.Del("Accept-Encoding") + resp, err := http.DefaultClient.Do(req) + if err != nil { + http.Error(w, "upstream failed", 502) + return + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + for k, values := range resp.Header { + for _, v := range values { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + _, _ = w.Write(body) + }) + listener, err := tbListenLoopback() + if err != nil { + t.Fatal(err) + } + srv := &http.Server{Handler: handler} + go func() { _ = srv.Serve(listener) }() + t.Cleanup(func() { _ = srv.Close() }) + p.url = "http://" + listener.Addr().String() + return p +} + +func (p *tbRefusingProxy) RefuseNextBulk() { p.refuse.Store(true) } +func (p *tbRefusingProxy) Disarm() { p.refuse.Store(false) } + +func tbDump(t *testing.T, ctx context.Context, pool *pgxpool.Pool) map[string][]string { + t.Helper() + dump := map[string][]string{} + tables := []string{"knowledges", "chunks", "knowledge_tags", "knowledge_tag_relations", "resources", + "resource_bindings", "wiki_folders", "wiki_pages", "wiki_page_issues", "wiki_log_entries"} + for _, table := range tables { + rows, err := pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table)) + if err != nil { + t.Fatal(err) + } + values := []string{} + for rows.Next() { + var raw string + if err := rows.Scan(&raw); err != nil { + rows.Close() + t.Fatal(err) + } + values = append(values, raw) + } + rows.Close() + dump[table] = values + } + return dump +} + +func tbDiffDump(before, after map[string][]string) string { + tables := make([]string, 0, len(before)) + for table := range before { + tables = append(tables, table) + } + sort.Strings(tables) + for _, table := range tables { + if len(before[table]) != len(after[table]) { + return fmt.Sprintf("%s rows %d -> %d", table, len(before[table]), len(after[table])) + } + for i := range before[table] { + if before[table][i] != after[table][i] { + return fmt.Sprintf("%s row %d changed: %s", table, i, after[table][i]) + } + } + } + return "" +} + +func tbIndexReadback(t *testing.T, ctx context.Context, env *tbEnv) map[string]string { + t.Helper() + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s/%s/_search?size=100", tbGetenv(t, "KG_LIBRARY_TEST_ELASTIC_URL"), env.index), + strings.NewReader(`{"query":{"match_all":{}},"size":100,"_source":true}`)) + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + var parsed struct { + Hits struct { + Hits []struct { + ID string `json:"_id"` + Source json.RawMessage `json:"_source"` + } `json:"hits"` + } `json:"hits"` + } + if err := json.Unmarshal(raw, &parsed); err != nil || resp.StatusCode != 200 { + t.Fatalf("index readback %d %s", resp.StatusCode, raw) + } + result := map[string]string{} + for _, hit := range parsed.Hits.Hits { + var source map[string]any + if err := json.Unmarshal(hit.Source, &source); err != nil { + t.Fatalf("index readback source %s: %v", hit.ID, err) + } + canonical, err := json.Marshal(source) + if err != nil { + t.Fatalf("index readback canonical %s: %v", hit.ID, err) + } + result[hit.ID] = string(canonical) + } + return result +} + +func tbDiffIndex(before, after map[string]string) string { + keys := map[string]bool{} + for k := range before { + keys[k] = true + } + for k := range after { + keys[k] = true + } + ordered := make([]string, 0, len(keys)) + for k := range keys { + ordered = append(ordered, k) + } + sort.Strings(ordered) + for _, k := range ordered { + if before[k] != after[k] { + return fmt.Sprintf("document %s changed: %q -> %q", k, before[k], after[k]) + } + } + return "" +} + +func tbIngressHealthy(t *testing.T, env *tbEnv) error { + t.Helper() + req, _ := http.NewRequest(http.MethodGet, env.ingress, nil) + req.Close = true + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + if resp.StatusCode != 200 { + return fmt.Errorf("ingress HTTP %d", resp.StatusCode) + } + return nil +} + +func tbAssertIngressHealthy(t *testing.T, env *tbEnv) { + t.Helper() + if err := tbIngressHealthy(t, env); err != nil { + t.Fatalf("ingress not restored: %v", err) + } +} + +func tbIngressClosed(env *tbEnv) error { + req, _ := http.NewRequest(http.MethodGet, env.ingress, nil) + req.Close = true + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil + } + defer resp.Body.Close() + return fmt.Errorf("ingress still answers HTTP %d", resp.StatusCode) +} + +func tbAssertMaintenanceReleased(t *testing.T, ctx context.Context, env *tbEnv) { + t.Helper() + inst, err := env.store.InstanceMaintenanceOf(ctx, env.project) + if err != nil { + t.Fatal(err) + } + if inst == nil || inst.ReleasedAt == nil { + t.Fatalf("instance maintenance not released: %#v", inst) + } + var released *time.Time + var state string + if err := env.gov.QueryRow(ctx, `SELECT released_at,state FROM library_maintenance WHERE scope='internal'`).Scan(&released, &state); err != nil { + t.Fatal(err) + } + if released == nil { + t.Fatalf("scope maintenance not released: state=%s", state) + } +} + +func tbFindDrillLedger(t *testing.T, ctx context.Context, env *tbEnv, jobID string) libraryrelease.WriteSet { + t.Helper() + var digest string + if err := env.gov.QueryRow(ctx, `SELECT evidence->>'writeset_digest' FROM library_versions + WHERE source_job_id=$1 AND kind='production_prebackup'`, jobID).Scan(&digest); err != nil { + t.Fatal(err) + } + if digest == "" { + t.Fatal("no persisted drill writeset digest") + } + set, err := v071.LoadWriteSet(env.archiveRoot, digest) + if err != nil { + t.Fatal(err) + } + return *set +} + +func tbListenLoopback() (net.Listener, error) { + return net.Listen("tcp", "127.0.0.1:0") +} diff --git a/internal/httpapi/library_verify.go b/internal/httpapi/library_verify.go index 1e72d9c..a742ab6 100644 --- a/internal/httpapi/library_verify.go +++ b/internal/httpapi/library_verify.go @@ -97,11 +97,25 @@ func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config) return } now := time.Now().UTC() + // 验证作业必须有和同步/恢复同一套不可变 operation:worker 的 owned attempt 从 + // job.operation_id 取身份,恢复派发闸口和对象网关都按它核验。没有 operation 的 + // verify 作业在真实恢复时会被 fail closed(审核 B2-R1)。 + intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobVerify, + "", digest, identity.Username, "", now) + if intentErr != nil { + if isIntentConflict(intentErr) { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict", + "error": "同一幂等键对应了不同的请求"}) + return + } + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify operation"}) + return + } job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{ ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope, Kind: governance.LibraryJobVerify, - IdempotencyKey: "library.verify:" + request.Scope + ":" + now.Format("20060102T150405.000000000"), - RequestDigest: digest, Actor: identity.Username, + IdempotencyKey: intent.JobKey, + RequestDigest: digest, OperationID: intent.Operation.ID, Actor: identity.Username, }, governance.VerifyStages(), false, now) if errors.Is(err, governance.ErrLibraryJobActive) { writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active", diff --git a/internal/httpapi/library_verify_test.go b/internal/httpapi/library_verify_test.go index 0e15593..a34d8ba 100644 --- a/internal/httpapi/library_verify_test.go +++ b/internal/httpapi/library_verify_test.go @@ -99,6 +99,14 @@ func TestLibraryVerifyEnqueuesVerifyJobWithCanaryStages(t *testing.T) { if !strings.HasPrefix(store.lastJob.ID, "libverify-internal-") { t.Fatalf("job id=%q", store.lastJob.ID) } + // B2-R1:verify 作业必须带不可变 operation,worker 的 owned attempt / 恢复派发 + // 身份都从它取;没有 operation 的作业在真实恢复时会被 fail closed。 + if store.lastJob.OperationID == "" || store.lastJob.OperationID != store.lastOperation.ID { + t.Fatalf("verify job must carry its operation: job=%#v op=%#v", store.lastJob, store.lastOperation) + } + if store.lastOperation.Kind != governance.LibraryJobVerify || store.lastOperation.Scope != "internal" { + t.Fatalf("verify operation identity=%#v", store.lastOperation) + } // 阶段必须包含 canary:没有"写 canary 证明校验有效"这一步,验证就只是自证。 want := governance.VerifyStages() if len(want) != 5 || want[2] != "canary" || want[3] != "restore" || want[4] != "finalize" { diff --git a/internal/libraryrelease/writeset.go b/internal/libraryrelease/writeset.go index 7ef2620..8a4c260 100644 --- a/internal/libraryrelease/writeset.go +++ b/internal/libraryrelease/writeset.go @@ -19,6 +19,12 @@ const WriteSetVersion = 1 // MigrationWriteSetVersion rejects reinterpretation by old V1 executors. const MigrationWriteSetVersion = 2 +// WriteSetKindRestoreDrill marks a write set that authorizes only a recovery +// drill: canary rows plus a zero-object/zero-index execution plan. It shares +// the V2 envelope so the object gateway controller can verify it, but it is +// never a publish plan and must not be adopted as one. +const WriteSetKindRestoreDrill = "restore-drill" + // WriteSetRow 是台账里的一行:本次发布明确要写的精确身份 + 写前状态。 // // 为什么需要"写前状态":恢复不能在目标里按"没人引用"推断残渣。历史孤儿资源、 @@ -89,6 +95,10 @@ type LocalIDMapping struct { } type WriteSet struct { + // Kind 区分发布执行计划与受限演练恢复计划。空值 = 历史发布计划。 + // 演练计划必须显式为 restore-drill:它只授权 canary 回放,不含对象/索引 + // 写项,读取方不能把它当发布计划接管。 + Kind string `json:"kind,omitempty"` Index []IndexWrite `json:"index,omitempty"` Objects []ObjectWrite `json:"objects,omitempty"` SchemaDigest string `json:"schema_digest,omitempty"` @@ -218,6 +228,22 @@ func (w *WriteSet) Verify() error { if w.Version != WriteSetVersion && w.Version != MigrationWriteSetVersion { return fmt.Errorf("写集合记录版本 %d 不受支持(当前 %d)", w.Version, WriteSetVersion) } + if w.Kind != "" && w.Kind != WriteSetKindRestoreDrill { + return fmt.Errorf("写集合记录种类 %q 不受支持", w.Kind) + } + if w.Kind == WriteSetKindRestoreDrill { + // 演练恢复计划只回放写前镜像:不新建对象、不按发布计划改索引、不做本地 ID 映射。 + // 任何一项非空都说明它被当成发布计划构造/改写过,拒绝。 + if w.Version != MigrationWriteSetVersion || w.ContractVersion != 1 || len(w.SchemaDigest) != 64 { + return errors.New("演练恢复计划缺少可执行契约(V2 + contract 1 + schema 摘要)") + } + if len(w.Objects) != 0 || len(w.Index) != 0 || len(w.LocalIDs) != 0 { + return errors.New("演练恢复计划不得包含对象/索引/本地映射写项") + } + if w.SourceTenantID == 0 || w.SourceKnowledgeBaseID == "" { + return errors.New("演练恢复计划缺少来源身份") + } + } if w.JobID == "" || w.BackupSnapshotID == "" || w.TargetSnapshotID == "" { return errors.New("写集合记录缺少作业/预备份/快照绑定") } @@ -248,6 +274,9 @@ func (w *WriteSet) Verify() error { return nil } +// IsRestoreDrill 表示这是一份受限演练恢复计划,而不是发布执行计划。 +func (w WriteSet) IsRestoreDrill() bool { return w.Kind == WriteSetKindRestoreDrill } + // CreatedIDs 返回"本次新建"的精确身份(写前不存在的行)。恢复只允许删这些。 func (w WriteSet) CreatedIDs(table string) []string { ids := make([]string, 0, 8) diff --git a/internal/libraryverify/canary_test.go b/internal/libraryverify/canary_test.go index af59222..c50f957 100644 --- a/internal/libraryverify/canary_test.go +++ b/internal/libraryverify/canary_test.go @@ -23,7 +23,7 @@ func TestCanaryPersistsAndRegistersEvidenceBeforeWrite(t *testing.T) { root := t.TempDir() ledger := drillLedger() registered, marked, written := false, false, false - err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", + err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", nil, func(context.Context) error { if !registered { t.Fatal("risk marker preceded durable registration") @@ -67,7 +67,7 @@ func TestCanaryNeverWritesWithoutDurableLedger(t *testing.T) { } ledger := drillLedger() marked, written := false, false - err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", + err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", nil, func(context.Context) error { marked = true; return nil }, func(context.Context) error { marked = true; return nil }, func(context.Context) error { written = true; return nil }) @@ -80,7 +80,7 @@ func TestCanaryNeverWritesWithoutDurableLedger(t *testing.T) { func TestCanaryNeverWritesWhenRegistrationFails(t *testing.T) { ledger := drillLedger() marked, written := false, false - err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", + err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", nil, func(context.Context) error { marked = true; return nil }, func(context.Context) error { return errors.New("governance registration unavailable") }, func(context.Context) error { written = true; return nil }) @@ -92,7 +92,7 @@ func TestCanaryNeverWritesWhenRegistrationFails(t *testing.T) { func TestCanaryNeverWritesWhenRiskMarkerFails(t *testing.T) { ledger := drillLedger() written := false - err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", + err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", nil, func(context.Context) error { return errors.New("journal unavailable") }, func(context.Context) error { return nil }, func(context.Context) error { written = true; return nil }) diff --git a/internal/libraryverify/verify.go b/internal/libraryverify/verify.go index 34d9d02..0b90dc8 100644 --- a/internal/libraryverify/verify.go +++ b/internal/libraryverify/verify.go @@ -252,6 +252,11 @@ func Run(ctx context.Context, config appconfig.Config, scope, jobID string, opts report.Canary = canary progress.report("canary", "running", "写入一个隔离 canary,验证校验能发现差异") if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary, + func(set *libraryrelease.WriteSet) error { + // 计划必须在 Seal/落盘/登记之前定型:对象网关控制器按登记摘要加载, + // 这里补 V2 信封、目标 schema、来源身份与当前 owned attempt 绑定。 + return target.PrepareDrillPlan(ctx, set) + }, func(ctx context.Context) error { if err := window.MarkContentRisk(ctx); err != nil { return err @@ -397,8 +402,14 @@ func ledgerClone(set *libraryrelease.WriteSet) *libraryrelease.WriteSet { // writeDrillCanary 是"写入前先留痕"的唯一实现:seal → 落盘 → 登记 → 标风险 → 写。 // 任何一步失败都不写目标;登记在标风险之前,恢复入口才拿得到这份写集合。 func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, + prepare func(*libraryrelease.WriteSet) error, mark func(context.Context) error, record func(context.Context) error, write func(context.Context) error) error { ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false}) + if prepare != nil { + if err := prepare(ledger); err != nil { + return err + } + } if err := ledger.Seal(); err != nil { return err } diff --git a/internal/releaseadapter/v071/gateway_object_store.go b/internal/releaseadapter/v071/gateway_object_store.go index b41ddc6..6edc48d 100644 --- a/internal/releaseadapter/v071/gateway_object_store.go +++ b/internal/releaseadapter/v071/gateway_object_store.go @@ -53,6 +53,12 @@ func ObjectGatewayPlan(set libraryrelease.WriteSet) (objectgateway.Plan, error) func gatewayCreateOperation(plan, owner string) string { return objectgateway.Digest([]byte(plan + "\x00" + owner + "\x00create")) } + +// OwnedAttemptClaim 暴露本装配冻结的 attempt claim:受限演练计划/回放必须与它一致。 +func (s *GatewayObjectStore) OwnedAttemptClaim() (objectgateway.Claim, bool) { + return s.Claim, s.Automatic +} + func (s *GatewayObjectStore) BindObjectPlan(set libraryrelease.WriteSet) error { if !validLowerSHA256(set.Digest) || set.JobID == "" { return errors.New("gateway frozen plan identity missing") diff --git a/internal/releaseadapter/v071/library_production_target.go b/internal/releaseadapter/v071/library_production_target.go index d37b275..4c48765 100644 --- a/internal/releaseadapter/v071/library_production_target.go +++ b/internal/releaseadapter/v071/library_production_target.go @@ -19,6 +19,7 @@ import ( "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarysnapshot" + "knowledge-governance/internal/objectgateway" "knowledge-governance/internal/testbarrier" ) @@ -116,6 +117,9 @@ type ProductionLibraryTarget struct { preserveChunkSequence bool activePlan *libraryrelease.WriteSet activeSnapshot string + // drillRestore 表示 activePlan 是受限演练恢复计划:索引按写前镜像重建, + // 对象只做只读字节核对(计划本身零对象写项)。发布恢复永不为 true。 + drillRestore bool // restoreScope / restoreSourceSnapshot / restoreBackupSnapshot 在 Restore 进入时 // 从**原持久锚点**冻结:scope 来自真实装配,来源快照与写前镜像来自本次 WriteSet // 与原 backup 引用。它们不能由调用方在恢复中途改写,也不允许同一个快照既当 @@ -681,7 +685,7 @@ WHERE tenant_id=$1 AND id = ANY($2::text[])`, t.scope.TenantID, ids) } func (t *ProductionLibraryTarget) importIndex(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error { - if t.activePlan != nil { + if t.activePlan != nil && !t.drillRestore { if t.preserveChunkSequence { return t.restorePlannedIndex(ctx) } @@ -927,6 +931,10 @@ func (t *ProductionLibraryTarget) AdoptWriteSet(set libraryrelease.WriteSet) err if err := set.Verify(); err != nil { return fmt.Errorf("写集合记录校验失败:%w", err) } + if set.IsRestoreDrill() { + // 演练计划只描述 canary 回放,不是发布执行计划;发布恢复入口不得消费它。 + return errors.New("演练恢复计划不能作为发布执行计划接管") + } if set.TenantID != t.scope.TenantID || set.KnowledgeBaseID != t.scope.KnowledgeBaseID { return fmt.Errorf("写集合绑定的是 %d/%s,与目标 %d/%s 不一致", set.TenantID, set.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID) @@ -983,6 +991,89 @@ func ValidateDrillWriteSet(set libraryrelease.WriteSet) error { return nil } +// PrepareDrillPlan 把演练台账升级为**受限可执行计划**:V2 执行信封 + 目标 schema +// 摘要 + 真实来源身份 + 当前 owned attempt 绑定,但对象/索引/本地映射写项必须为空。 +// +// 为什么必须在这里而不是在 Apply 里临时补:对象网关控制器只认治理库里 +// production_prebackup 证据登记的 writeset_digest;计划必须在 Seal/落盘/登记之前 +// 定型,否则控制器加载到的摘要是另一份形状。它只增加受限语义,不放宽任何发布校验。 +func (t *ProductionLibraryTarget) PrepareDrillPlan(ctx context.Context, set *libraryrelease.WriteSet) error { + if set == nil || set.JobID == "" { + return errors.New("演练恢复计划缺少作业身份") + } + if set.TenantID != t.scope.TenantID || set.KnowledgeBaseID != t.scope.KnowledgeBaseID { + return fmt.Errorf("演练恢复计划绑定的是 %d/%s,与目标 %d/%s 不一致", + set.TenantID, set.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID) + } + if err := set.ValidateRestoreBinding(set.BackupSnapshotID, t.scope.TenantID, t.scope.KnowledgeBaseID); err != nil { + return err + } + archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) + if err != nil { + return errors.New("归档不可用") + } + source, err := archive.Load(set.TargetSnapshotID) + archive.Close() + if err != nil { + return errors.New("演练来源快照不可读") + } + if source.TenantID != t.scope.TenantID || source.KnowledgeBaseID != t.scope.KnowledgeBaseID { + return errors.New("演练来源快照不属于当前目标范围") + } + _, schemaDigest, err := t.migrationCatalog(ctx) + if err != nil { + return err + } + // 先清空任何发布写项:演练计划只允许 canary 行。 + set.Kind = libraryrelease.WriteSetKindRestoreDrill + set.Version = libraryrelease.MigrationWriteSetVersion + set.ContractVersion = migrationIdentityContractVersion + set.SourceTenantID, set.SourceKnowledgeBaseID = source.TenantID, source.KnowledgeBaseID + set.SchemaDigest = schemaDigest + set.Index, set.Objects, set.LocalIDs = nil, nil, nil + // 自动对象网关没有静态 grant:计划签发必须绑定当前 owned attempt, + // 网关控制器会再按治理库核一次(不靠调用方自证)。 + if claim, automatic := t.ownedAttempt(); automatic { + return validateDrillAttempt(claim, set.JobID) + } + return nil +} + +// ownedAttempt 返回本目标装配时冻结的对象网关 claim。只有 Automatic 网关需要它。 +func (t *ProductionLibraryTarget) ownedAttempt() (objectgateway.Claim, bool) { + if provider, ok := t.config.Objects.(interface { + OwnedAttemptClaim() (objectgateway.Claim, bool) + }); ok { + return provider.OwnedAttemptClaim() + } + return objectgateway.Claim{}, false +} + +// validateDrillAttempt 要求计划/回放绑定一个真实的作业代次,而不是调用方编造的身份。 +func validateDrillAttempt(claim objectgateway.Claim, jobID string) error { + if claim.JobID == "" || claim.JobID != jobID || claim.OperationID == "" || claim.Owner == "" || claim.AttemptEpoch < 1 { + return errors.New("演练恢复缺少当前 owned attempt 身份(job/operation/owner/epoch)") + } + return nil +} + +// validateDrillDispatch 证明这次演练的派发身份来自原窗口的真实 attempt 与恢复投递, +// 不是把 gate 置 nil 绕过 DS01。 +func (t *ProductionLibraryTarget) validateDrillDispatch(claim objectgateway.Claim, automatic bool) error { + config := t.config + if config.RestoreTargetIdentity == "" || config.RestoreScope == "" || config.RestoreInstance == "" || + config.RestoreOperationID == "" || config.RestoreWindowJobID == "" || config.RestoreAttemptOwner == "" || + config.RestoreAttemptEpoch < 1 || config.RestoreRecoveryJobID == "" || config.RestoreRecoveryOperationID == "" { + return errors.New("演练恢复缺少完整的派发身份(目标/scope/实例/原 operation/窗口/attempt/恢复作业)") + } + if automatic && (claim.JobID != config.RestoreWindowJobID || claim.OperationID != config.RestoreOperationID || + claim.Owner != config.RestoreAttemptOwner || claim.AttemptEpoch != config.RestoreAttemptEpoch || + claim.RecoveryJobID != config.RestoreRecoveryJobID || claim.RecoveryOperationID != config.RestoreRecoveryOperationID) { + return errors.New("演练恢复的派发身份与当前 owned attempt 不一致") + } + return nil +} + // RestoreDrill 是恢复演练**限定范围**的安全恢复入口。 // // 为什么不是直接调用 Restore:Restore 的写集合是"发布执行计划"(v2 合同 + 索引/对象/ @@ -990,7 +1081,10 @@ func ValidateDrillWriteSet(set libraryrelease.WriteSet) error { // 一份从未发生过的发布。这里保留同一份 DS01 身份合同(来源快照 ≠ 写前镜像、同一作业/ // 目标/预备份)与同一批 Apply 原语:四个阶段都按写前镜像**整段回放**(先清 scope 再 // 插入归档行),不是"只删 canary";回放后再用整实例指纹证明目标与预备份一致。 -// 有派发闸口的目标不走这条入口:那条路径必须有过派发证据的恢复计划。 +// +// 自动对象网关保持启用:演练计划显式零对象写项,仍由控制器按当前 attempt/job/目标/ +// 写前镜像签发零写入许可;对象阶段只做只读字节核对,任何创建/删除都会被网关拒绝。 +// 有派发闸口的目标也不再把 gate 置 nil:必须携带原窗口 attempt + 恢复投递身份。 func (t *ProductionLibraryTarget) RestoreDrill(ctx context.Context, backup libraryrelease.BackupRef) error { if backup.SnapshotID == "" { return errors.New("恢复需要绑定的预备份快照") @@ -1007,8 +1101,43 @@ func (t *ProductionLibraryTarget) RestoreDrill(ctx context.Context, backup libra if err := ValidateDrillWriteSet(*backup.WriteSet); err != nil { return err } + if !backup.WriteSet.IsRestoreDrill() { + return errors.New("演练恢复只接受受限恢复计划(restore-drill),拒绝发布执行计划") + } + claim, automatic := t.ownedAttempt() + if automatic { + if err := validateDrillAttempt(claim, backup.WriteSet.JobID); err != nil { + return err + } + } if t.config.RestoreDispatch != nil { - return errors.New("目标配置了恢复派发闸口:演练不能用绕过派发证据的回放入口") + if backup.WriteSet.Target != t.config.RestoreScope { + return fmt.Errorf("演练恢复计划 scope=%s,与派发身份 scope=%s 不一致", + backup.WriteSet.Target, t.config.RestoreScope) + } + if err := t.validateDrillDispatch(claim, automatic); err != nil { + return err + } + } + // 对象网关:冻结零对象计划。控制器仍按治理库核 attempt/job/目标/写前镜像; + // 没有冻结计划时 EnsureObjectGrant 会继续 fail closed。 + if binder, ok := t.config.Objects.(interface { + BindObjectPlan(libraryrelease.WriteSet) error + }); ok { + if err := binder.BindObjectPlan(*backup.WriteSet); err != nil { + return err + } + } + adopted := *backup.WriteSet + adopted.Rows = append([]libraryrelease.WriteSetRow(nil), backup.WriteSet.Rows...) + t.activePlan = &adopted + t.drillRestore = true + // DS01 准入必须在任何 PG/对象/索引写之前:原窗口若还有未决派发记录, + // 即使计划是零对象也要 held。 + if t.config.RestoreDispatch != nil { + if err := t.admitRestorePlan(ctx); err != nil { + return err + } } // 幂等:恢复收尾可能重放(上次恢复到一半崩溃)。这里不要求"目标当前仍与镜像不同"—— // "写确实发生"由发起方在写入后的 drift 校验证明,而不是由重放入口猜。 @@ -1020,6 +1149,16 @@ func (t *ProductionLibraryTarget) RestoreDrill(ctx context.Context, backup libra return fmt.Errorf("演练恢复阶段 %s 失败: %w", stage, err) } } + // canary 是这次演练唯一新建的行:回放后必须精确消失(不是"整库看起来差不多")。 + for _, id := range backup.WriteSet.CreatedIDs("knowledge_tags") { + var exists bool + if err := t.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)`, id).Scan(&exists); err != nil { + return fmt.Errorf("读取演练 canary 残留失败: %w", err) + } + if exists { + return fmt.Errorf("演练恢复后 canary %s 仍在目标中", id) + } + } after, err := universeFingerprint(ctx, t.pool) if err != nil { return fmt.Errorf("演练恢复后读取整库指纹失败: %w", err) @@ -1045,6 +1184,9 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel if err := backup.WriteSet.Verify(); err != nil { return fmt.Errorf("写集合记录校验失败,拒绝恢复:%w", err) } + if backup.WriteSet.IsRestoreDrill() { + return errors.New("演练恢复计划不能走发布恢复入口:请使用同作业的恢复演练路径") + } // 身份合同与演练共用同一份判定(libraryrelease.WriteSet.ValidateRestoreBinding): // 来源快照 ≠ 写前镜像、同一预备份、同一 tenant/KB,缺一不可。 if err := backup.WriteSet.ValidateRestoreBinding(backup.SnapshotID, t.scope.TenantID, t.scope.KnowledgeBaseID); err != nil { diff --git a/scripts/test-scope.json b/scripts/test-scope.json index b6c8a40..d64bdf1 100644 --- a/scripts/test-scope.json +++ b/scripts/test-scope.json @@ -430,6 +430,7 @@ "reason": "真实隔离PG/ES/MinIO;验证mapping哈希不变和成功后无暂态marker,默认CI无这些服务" }, {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestCandidateEmptyTargetFixture","reason":"需显式一次性本机PG/ES/MinIO与运行器注入的隔离库/归档根:为候选页面装配并封存独立 fullshape 测试来源、证空目标、登记 test_sealed;普通CI跳过不是通过"}, - {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestCandidateAttestation","reason":"需运行器注入一次性 ed25519 种子与四份真实证据摘要:续签 object-gateway 审计(有效期上限1h);普通CI没有注入,跳过不是通过"} + {"package":"knowledge-governance/internal/releaseadapter/v071","test":"TestCandidateAttestation","reason":"需运行器注入一次性 ed25519 种子与四份真实证据摘要:续签 object-gateway 审计(有效期上限1h);普通CI没有注入,跳过不是通过"}, + {"package":"knowledge-governance/cmd/worker","test":"TestThreeBlockersVerifyChain","reason":"需外置 runner 创建的一次性隔离 PG/ES/MinIO 加真实入口/网关容器与 Automatic+DS01 授权:完整 verify 与同 job recover 纵向链;普通CI没有这些资源,跳过不是通过"} ] } diff --git a/web/src/RestoreVerificationSection.mounted.test.tsx b/web/src/RestoreVerificationSection.mounted.test.tsx index 7cbf9e2..c749835 100644 --- a/web/src/RestoreVerificationSection.mounted.test.tsx +++ b/web/src/RestoreVerificationSection.mounted.test.tsx @@ -158,6 +158,47 @@ describe('RestoreVerificationSection mounted (offline fixtures)', () => { expect(onComplete).toHaveBeenCalledWith('libverify-internal-1') }) + it('切 scope 后迟到的旧 POST 不能污染新目标或触发父级旧作业回调', async () => { + const onAccepted = vi.fn() + const onComplete = vi.fn() + let resolvePost: ((value: unknown) => void) | null = null + vi.stubGlobal('fetch', async (url: string, init?: { method?: string }) => { + if (init?.method === 'POST') { + return new Promise((resolve) => { + resolvePost = resolve + }) + } + return jsonResponse(String(url).includes('scope=external') ? externalView : internalView) + }) + await act(async () => { + root.render( + + ) + }) + // 旧 internal POST 挂起时切到 external:external 的真实观测已经完成。 + await act(async () => { + ;[...container.querySelectorAll('button')].find((item) => item.textContent?.includes('跑一次恢复验证'))!.click() + }) + await act(async () => { + root.render( + + ) + }) + expect(container.textContent).toContain('fixture-external') + expect(container.textContent).not.toContain('已验证(当前目标)') + expect(container.textContent).not.toContain('old-internal-job') + // 放行旧 internal POST:它的 then/onAccepted/reload 必须全部失效。 + await act(async () => { + resolvePost?.(jsonResponse({ job: { id: 'verify-internal-new', scope: 'internal' }, reused: false }, 202)) + }) + expect(container.textContent).toContain('fixture-external') + expect(container.textContent).not.toContain('已验证(当前目标)') + expect(container.textContent).not.toContain('old-internal-job') + expect(container.textContent).not.toContain('verify-internal-new') + expect(onAccepted).not.toHaveBeenCalled() + expect(onComplete).not.toHaveBeenCalled() + }) + it('HTTP 失败按失败显示,不把响应当通过证据', async () => { vi.stubGlobal('fetch', async () => jsonResponse({ scope: 'internal', verified: true, reason: '证据读取失败', target, history: [] }, 503) diff --git a/web/src/RestoreVerificationSection.tsx b/web/src/RestoreVerificationSection.tsx index 5055711..c53d51f 100644 --- a/web/src/RestoreVerificationSection.tsx +++ b/web/src/RestoreVerificationSection.tsx @@ -112,8 +112,15 @@ export function RestoreVerificationSection({ const mounted = useRef(true) const generation = useRef(0) const viewScopeRef = useRef('') + // scopeRef 是"当前渲染的 scope":异步回调必须同时核对 scope 与世代, + // 只比 GET generation 会被迟到的旧 POST 重新递增而失效(审核 B5-R1)。 + const scopeRef = useRef(scope) + const scopeEpoch = useRef(0) + const actionSeq = useRef(0) const acceptedJob = useRef('') const sawRunning = useRef(false) + const onAcceptedRef = useRef(onAccepted) + onAcceptedRef.current = onAccepted const onCompleteRef = useRef(onComplete) onCompleteRef.current = onComplete @@ -122,7 +129,7 @@ export function RestoreVerificationSection({ return fetchLibraryVerification(targetScope, signal) .then((next) => { // 晚到的旧 scope/旧代次响应一律丢弃:它不能改写当前 scope 的观测。 - if (!mounted.current || request !== generation.current) return + if (!mounted.current || request !== generation.current || targetScope !== scopeRef.current) return setView(next) viewScopeRef.current = next.scope setStale(false) @@ -141,7 +148,7 @@ export function RestoreVerificationSection({ } }) .catch((err: unknown) => { - if (!mounted.current || request !== generation.current) return + if (!mounted.current || request !== generation.current || targetScope !== scopeRef.current) return if ((err as Error)?.name === 'AbortError') return // 刷新失败:同 scope 的旧结果保留但明确标成过期;别的 scope 的旧结果直接丢掉。 setStale(true) @@ -152,6 +159,11 @@ export function RestoreVerificationSection({ useEffect(() => { mounted.current = true + scopeRef.current = scope + // 切 scope 使所有在飞动作失效:旧 scope 的 POST/回调不能写回新目标。 + scopeEpoch.current += 1 + actionSeq.current = 0 + setPending(false) // 切 scope:旧 scope 的证据不能继续显示为"当前目标已验证"。 setView(null) viewScopeRef.current = '' @@ -173,21 +185,37 @@ export function RestoreVerificationSection({ }, [scope]) const onRun = useCallback(() => { + const actionScope = scope + const actionEpoch = scopeEpoch.current + const action = ++actionSeq.current + // 动作仍然有效 = 组件挂着 + 还是同一个 scope 世代 + 仍是最新一次点击。 + const current = () => + mounted.current && + scopeEpoch.current === actionEpoch && + scopeRef.current === actionScope && + actionSeq.current === action setPending(true) setError(null) setNotice('') - enqueueLibraryVerification(scope) + enqueueLibraryVerification(actionScope) .then((accepted) => { + if (!current() || accepted.scope !== actionScope) return acceptedJob.current = accepted.job_id sawRunning.current = false setNotice(`已投递恢复验证作业 ${accepted.job_id};页面会自动刷新进度与终态,父级作业/维护/闸口同步更新。`) // 父页面必须立刻知道"有一个验证在跑":否则它不会轮询 overview/维护。 - onAccepted?.(accepted.job_id) - void reload(scope) + onAcceptedRef.current?.(accepted.job_id) + void reload(actionScope) }) - .catch((err: unknown) => setError(err instanceof Error ? err.message : '投递恢复验证失败')) - .finally(() => setPending(false)) - }, [scope, reload, onAccepted]) + .catch((err: unknown) => { + if (!current()) return + setError(err instanceof Error ? err.message : '投递恢复验证失败') + }) + .finally(() => { + // 旧动作的 finally 不能清掉新 scope 新动作的 pending。 + if (current()) setPending(false) + }) + }, [scope, reload]) return (