diff --git a/CHANGELOG.md b/CHANGELOG.md index ace3a4c..30731d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 开发快照(正式 worker / 维护恢复入口的 held → 恢复闭环验收) + +### Added + +- 新增正式链路验收 `TestFormalMaintenanceRecoveryHeldThenRecovered`(只在 `KG_R38_RUN` 指定的 + 一次性隔离部署里运行,常规运行与 CI 跳过):真实 publish 在 database/resources/index/wiki 四类 + 真实写入完成后,由"校验读回内容与快照不一致"失败进入自动回滚;回滚在 + `restore-finalize-intent` 已落盘、marker 仍停在自己世代时被传输层拒绝 clean 派发(拒绝 = 不转发、 + 不丢响应),窗口保持 held,实例/scope 两个维护记录写 `needs_restore`,写入口返回 409;撤掉一次性 + 故障后 `POST /api/v1/library/maintenance/recover` 以同一 window/job/backup 续作成功,PG 逐行、 + 对象版本集合与 ES `_source` 全量恢复,入口重新返回 200。 +- 该用例中的对象 receipt 全部由真实对象网关写入产生(committed/removed 两条不可变回执 + 对应 + delivery/plan 摘要),没有人工造回执,也没有改核心状态。 + +### Changed + +- 本轮只新增验收与运行器,产品行为、协议与判据均未改变;删除型恢复的 R02-b 缺口保持原样。 + ## 开发快照(删除型索引恢复不再把 ES 统计计数器当证据) ### Fixed diff --git a/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go b/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go new file mode 100644 index 0000000..8e03e25 --- /dev/null +++ b/internal/releaseadapter/v071/formal_maintenance_recovery_r38_test.go @@ -0,0 +1,990 @@ +package v071 + +// r38:正式 worker / maintenance / recover API 的"有证据可恢复"闭环。 +// +// 场景(全部由真实进程与真实存储产生,测试只播种/观察): +// 1. 真实 publish:worker 执行 apply(database/resources/index 真实写入,对象经对象网关 +// 生成 receipt);快照缺 wiki 部件 → wiki 阶段真实失败(与 r37 contractFailWikiTarget 同一定点)。 +// 2. 失败后的自动回滚在 restore-finalize-intent 已落盘、marker 仍停在自己世代时被 +// 传输层拒绝"clean 派发"(拒绝 = 不转发、不丢响应;ES 里 marker 原样保留)。 +// → 回滚失败 → 维护窗口 held + needs_restore(不是 adapter 返回值当结论)。 +// 3. HTTP:GET /library/maintenance 看到 held/needs_restore;写入口 POST /library/restores 返回 409。 +// 4. 撤一次性故障 → POST /library/maintenance/recover(同一 window/job/backup)→ worker 续作 +// 一次 clean 写入 → 校验通过、入口 open、维护释放;PG 逐行/对象版本/ES _source 全量恢复。 +// +// 该测试只在 KG_R38_RUN 指定的隔离部署里运行;故障只存在于本进程的 ES 传输层。 + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/cookiejar" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/jackc/pgx/v5/pgxpool" + "golang.org/x/crypto/bcrypt" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" + "knowledge-governance/internal/objectgateway" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// r38RefusalProxy 是"只在传输层拒绝,不转发"的故障注入: +// 选中的写请求被直接 503,绝不落到真实 ES;其余请求原样转发并记录变更请求现场。 +type r38RefusalProxy struct { + url string + mu sync.Mutex + refuseClean bool + corruptVerify bool + events []map[string]any + refused int + corrupted int + root string +} + +func newR38RefusalProxy(t *testing.T, root, upstream string) *r38RefusalProxy { + t.Helper() + p := &r38RefusalProxy{root: root, refuseClean: true} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, e := io.ReadAll(r.Body) + if e != nil { + http.Error(w, "read failed", 500) + return + } + p.mu.Lock() + refuseClean := p.refuseClean + p.mu.Unlock() + selected := refuseClean && r.Method == http.MethodPut && + strings.Contains(r.URL.Path, "/_doc/chunk-target") && + !bytes.Contains(raw, []byte(indexRestoreClaimPrefix)) + if selected { + p.mu.Lock() + p.refused++ + p.events = append(p.events, map[string]any{ + "method": r.Method, "path": r.URL.RequestURI(), "request": string(raw), + "dispatched": false, "refused": true, "at": time.Now().UTC().Format(time.RFC3339Nano)}) + p.flush() + p.mu.Unlock() + // 拒绝发生在派发之前:真实 ES 一次都没有收到这份 clean 写入。 + http.Error(w, "r38 injected refusal before clean dispatch", http.StatusServiceUnavailable) + return + } + request, e := http.NewRequestWithContext(r.Context(), r.Method, upstream+r.URL.RequestURI(), bytes.NewReader(raw)) + if e != nil { + http.Error(w, "request failed", 500) + return + } + request.Header = r.Header.Clone() + // 读回内容必须是明文 JSON:否则注入的"内容不一致"只会变成无法解析的压缩字节, + // 校验侧会把它当成传输错误而不是真实内容差异。 + request.Header.Del("Accept-Encoding") + response, e := http.DefaultClient.Do(request) + if e != nil { + http.Error(w, "upstream failed", 502) + return + } + body, e := io.ReadAll(response.Body) + response.Body.Close() + if e != nil { + http.Error(w, "upstream body failed", 502) + return + } + p.mu.Lock() + corrupt := p.corruptVerify && response.StatusCode == 200 && strings.Contains(r.URL.Path, "_search") + if corrupt { + // 只改这一份读回内容并立刻解除:校验阶段读到"与快照不一致"的真实不一致, + // 后续回滚/恢复的 _mget 与 _doc 请求仍然读到真实 ES。 + p.corruptVerify = false + p.corrupted++ + body = corruptSearchBody(body) + } + p.mu.Unlock() + if r.Method == http.MethodPut || r.Method == http.MethodPost || r.Method == http.MethodDelete { + p.mu.Lock() + p.events = append(p.events, map[string]any{ + "method": r.Method, "path": r.URL.RequestURI(), "request": string(raw), + "status": response.StatusCode, "dispatched": true, "refused": false, "corrupted": corrupt, + "at": time.Now().UTC().Format(time.RFC3339Nano)}) + if len(p.events)%8 == 0 { + p.flush() + } + p.mu.Unlock() + } + for k, values := range response.Header { + for _, v := range values { + w.Header().Add(k, v) + } + } + w.WriteHeader(response.StatusCode) + _, _ = w.Write(body) + })) + t.Cleanup(server.Close) + p.url = server.URL + p.mu.Lock() + p.flush() + p.mu.Unlock() + return p +} + +func (p *r38RefusalProxy) flush() { + encoded, _ := json.MarshalIndent(p.events, "", " ") + _ = os.WriteFile(filepath.Join(p.root, "es-wire.json"), encoded, 0600) +} + +func (p *r38RefusalProxy) setRefuseClean(refuse bool) { + p.mu.Lock() + defer p.mu.Unlock() + p.refuseClean = refuse + p.flush() +} + +func (p *r38RefusalProxy) armVerifyCorruption() { + p.mu.Lock() + defer p.mu.Unlock() + p.corruptVerify = true + p.flush() +} + +func (p *r38RefusalProxy) corruptionCount() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.corrupted +} + +// corruptSearchBody 让校验阶段的读回与快照内容不一致(真实内容差异,不是错误响应)。 +func corruptSearchBody(body []byte) []byte { + var payload map[string]any + if json.Unmarshal(body, &payload) != nil { + return body + } + hits, _ := payload["hits"].(map[string]any)["hits"].([]any) + if len(hits) == 0 { + return body + } + hit, _ := hits[0].(map[string]any) + source, _ := hit["_source"].(map[string]any) + if source == nil { + return body + } + content, _ := source["content"].(string) + source["content"] = content + "-r38-verify-mismatch" + encoded, e := json.Marshal(payload) + if e != nil { + return body + } + return encoded +} + +func (p *r38RefusalProxy) snapshot() ([]map[string]any, int) { + p.mu.Lock() + defer p.mu.Unlock() + out := append([]map[string]any(nil), p.events...) + return out, p.refused +} + +type r38API struct { + base string + client *http.Client +} + +func (a r38API) do(t *testing.T, method, path string, payload any) (int, map[string]any) { + t.Helper() + var body []byte + if payload != nil { + encoded, e := json.Marshal(payload) + if e != nil { + t.Fatal(e) + } + body = encoded + } + request, e := http.NewRequest(method, a.base+path, bytes.NewReader(body)) + if e != nil { + t.Fatal(e) + } + if body != nil { + request.Header.Set("Content-Type", "application/json") + } + response, e := a.client.Do(request) + if e != nil { + t.Fatalf("%s %s: %v", method, path, e) + } + defer response.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + out := map[string]any{} + if len(raw) > 0 { + _ = json.Unmarshal(raw, &out) + } + return response.StatusCode, out +} + +func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) { + root := os.Getenv("KG_R38_RUN") + if root == "" { + t.Skip("requires the dedicated r38 isolated deployment (KG_R38_RUN)") + } + ctx := context.Background() + save := func(name string, value any) { + t.Helper() + raw, e := json.MarshalIndent(value, "", " ") + if e != nil { + t.Fatal(e) + } + if e = os.WriteFile(filepath.Join(root, name), raw, 0600); e != nil { + t.Fatal(e) + } + } + must := func(e error) { + t.Helper() + if e != nil { + t.Fatal(e) + } + } + archiveRoot := filepath.Join(root, "archive") + must(os.MkdirAll(archiveRoot, 0700)) + must(os.MkdirAll(filepath.Join(root, "maintenance"), 0700)) + + // ---------- 1) 隔离目标现场 ---------- + fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true, archiveRoot: archiveRoot, embeddingDimension: 1024}) + seedTargetData(t, fx) + _, e := fx.pool.Exec(ctx, `ALTER TABLE knowledge_bases ADD COLUMN embedding_model_id text, ADD COLUMN vector_store_id text; +CREATE TABLE models(id text PRIMARY KEY,name text,parameters jsonb,deleted_at timestamptz); +CREATE TABLE vector_stores(id text PRIMARY KEY,engine_type text,index_config jsonb,deleted_at timestamptz); +INSERT INTO models VALUES('r38-model','text-embedding-v4','{"embedding_parameters":{"dimension":1024}}',NULL); +INSERT INTO vector_stores VALUES('r38-vector','elasticsearch_v8','{}',NULL); +UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-vector';`) + must(e) + beforePG := universeDump(t, fx.pool) + save("01-pg-before.json", beforePG) + beforeIndex := recoveryIndexReadback(t, fx) + save("02-index-before.json", beforeIndex) + beforeObjects := objectInventoryR38(t, ctx, fx) + save("03-objects-before.json", beforeObjects) + + target := fx.target(t) + mapping, e := target.Search().MappingSHA256(ctx) + must(e) + // 快照完整、可封存;发布在四个阶段真实写完之后由"读回内容与快照不一致"的校验失败 + // 进入回滚(与 r37 在 wiki 阶段制造失败后回滚同一定点,但这里连 wiki 也是真实写入的)。 + source := sealSourceSnapshot(t, fx, nil) + archive, e := librarysnapshot.Open(archiveRoot, 256<<20) + must(e) + manifest, e := archive.Load(source) + must(e) + manifest.Compatibility.EmbeddingDimension = 1024 + manifest.Compatibility.IndexMappingSHA256 = mapping + source, e = archive.Seal(manifest) + must(e) + archive.Close() + save("04-snapshot.json", map[string]any{"snapshot": source, "index_mapping_sha256": mapping}) + + // ---------- 2) 治理库 / 网关 / API / worker ---------- + govDSN := os.Getenv("KG_R38_GOVERNANCE_DSN") + gov, e := pgxpool.New(ctx, govDSN) + must(e) + t.Cleanup(gov.Close) + must(postgresstore.ApplyMigrations(ctx, gov)) + _, e = gov.Exec(ctx, objectgateway.Schema) + must(e) + reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN) + project := os.Getenv("KG_R38_PROJECT") + proxy := newR38RefusalProxy(t, root, fx.searchBase) + + env := map[string]string{ + "KG_LIBRARY_TEST_ISOLATED": "1", "DATABASE_URL": govDSN, "LIBRARY_WORKER_ROLE": "release-executor", + "WORKER_POLL_INTERVAL": "200ms", "WORKER_LEASE_DURATION": "30s", + "PRODUCTION_RELEASE_REQUESTS_ENABLED": "true", "PRODUCTION_BACKUP_RESTORE_VERIFIED": "true", + "LIBRARY_ARCHIVE_ROOT": archiveRoot, "LIBRARY_CAPACITY_BUDGET_BYTES": "1073741824", + "LIBRARY_PRODUCTION_DATABASE_URL": os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), + "LIBRARY_PRODUCTION_ELASTIC_URL": proxy.url, "LIBRARY_PRODUCTION_INDEX": fx.index, + "LIBRARY_PRODUCTION_TENANT_ID": "10001", "LIBRARY_PRODUCTION_KB_INTERNAL": wsTargetKB, + "LIBRARY_PRODUCTION_S3_PROVIDER": "minio", "LIBRARY_PRODUCTION_S3_ENDPOINT": os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT"), + "LIBRARY_PRODUCTION_S3_BUCKET": fx.bucket, + "LIBRARY_PRODUCTION_S3_ACCESS_KEY": os.Getenv("KG_R38_READER_KEY"), + "LIBRARY_PRODUCTION_S3_SECRET_KEY": os.Getenv("KG_R38_READER_SECRET"), + "LIBRARY_TARGET_INSTANCE": project, "LIBRARY_TARGET_PROJECT": project, "LIBRARY_TARGET_WRITERS": "writer", + "LIBRARY_TARGET_NON_WRITERS": "ingress=entry gate,redis=queue,minio=object backend", + "LIBRARY_TARGET_INGRESS_URL": os.Getenv("KG_R38_INGRESS"), "LIBRARY_TARGET_INGRESS_SERVICE": "ingress", + "LIBRARY_TARGET_INGRESS_DRAIN_FILE": filepath.Join(root, "gate/drain.json"), + "LIBRARY_TARGET_REDIS_ADDR": os.Getenv("KG_R38_REDIS_ADDR"), "LIBRARY_TARGET_REDIS_PASSWORD": os.Getenv("KG_R38_REDIS_PASSWORD"), + "LIBRARY_TARGET_REDIS_DB": "0", "LIBRARY_TARGET_JOURNAL_ROOT": filepath.Join(root, "maintenance"), + "LIBRARY_TARGET_DOCKER_SOCKET": "/var/run/docker.sock", "LIBRARY_EMBEDDING_DIMENSION": "1024", + "KG_TEST_BARRIER_DIR": filepath.Join(root, "barriers"), "KG_TEST_BARRIER_POINTS": "after:wiki", + "KG_TEST_BARRIER_TIMEOUT": "300s", + } + env["PRODUCTION_RELEASE_AUTHORIZER_REF"] = project + env["PRODUCTION_RELEASE_AUTH_TOKEN_FILE"] = filepath.Join(root, "authorization-token") + must(os.WriteFile(env["PRODUCTION_RELEASE_AUTH_TOKEN_FILE"], []byte(objectgateway.Digest([]byte(root+"approval"))), 0600)) + cfg, e := appconfig.Load(func(k string) string { return env[k] }) + must(e) + dataToken := objectgateway.Digest([]byte(root + "data")) + controlToken := objectgateway.Digest([]byte(root + "control")) + env["LIBRARY_OBJECT_GATEWAY_URL"] = "http://" + os.Getenv("KG_R38_GATEWAY_ADDR") + env["LIBRARY_OBJECT_GATEWAY_TOKEN"] = dataToken + + // worker 的对象身份是只读的:任何对象写入只能经网关,receipt 因此必然来自真实生产写。 + readStore, e := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: env["LIBRARY_PRODUCTION_S3_ENDPOINT"], + Bucket: fx.bucket, AccessKeyID: env["LIBRARY_PRODUCTION_S3_ACCESS_KEY"], SecretAccessKey: env["LIBRARY_PRODUCTION_S3_SECRET_KEY"], + Region: "us-east-1", ForcePathStyle: true}) + must(e) + allowedRead, e := readStore.Open(ctx, "minio", wsLocator("backend-target", "minio", fx.bucket, 10001, "res-target")) + must(e) + allowedBody, e := io.ReadAll(allowedRead) + allowedRead.Close() + must(e) + if !bytes.Equal(allowedBody, []byte("12345678")) { + t.Fatal("readonly identity could not read original bytes") + } + if denied := readStore.Put(ctx, "minio", wsLocator("backend-target", "minio", fx.bucket, 10001, "r38-denied"), bytes.NewReader([]byte("denied")), 6); denied == nil { + t.Fatal("worker readonly identity unexpectedly wrote an object") + } + save("05-identity.json", map[string]any{"worker_object_identity": "readonly", "gateway_only_writer": true, + "bucket": fx.bucket, "index": fx.index, "target": cfg.LibraryTargetIdentity(), "project": project, + "readonly_read_bytes": string(allowedBody), "readonly_put_denied": true}) + + scope := objectgateway.Digest([]byte("minio\x00" + env["LIBRARY_PRODUCTION_S3_ENDPOINT"] + "\x00" + fx.bucket)) + pub, priv, e := ed25519.GenerateKey(rand.Reader) + must(e) + digestFile := func(name string) string { + raw, e := os.ReadFile(filepath.Join(root, name)) + must(e) + return objectgateway.Digest(raw) + } + initialDrain := filepath.Join(root, "initial-drain.json") + if _, e := os.Stat(initialDrain); e != nil { + t.Fatalf("runner must provide real initial drain evidence: %v", e) + } + payload, e := json.Marshal(objectgateway.Attestation{Protocol: objectgateway.Protocol, Scope: scope, GatewayID: project, + IssuedAt: time.Now().Add(-time.Second), ExpiresAt: time.Now().Add(50 * time.Minute), + WriterInventory: digestFile("05-identity.json"), PolicyEvidence: digestFile("05-identity.json"), + DenialEvidence: digestFile("05-identity.json"), DrainEvidence: digestFile("initial-drain.json")}) + must(e) + signedRaw, e := json.Marshal(objectgateway.SignedAttestation{Payload: payload, Signature: ed25519.Sign(priv, payload)}) + must(e) + // 审计有效期按真实时钟记录(网关要求不超过 1 小时;这里自己证明没有改系统时间)。 + var attestationView objectgateway.Attestation + must(json.Unmarshal(payload, &attestationView)) + nowProbe := time.Now() + if attestationView.ExpiresAt.Sub(attestationView.IssuedAt) > time.Hour || nowProbe.After(attestationView.ExpiresAt) { + t.Fatalf("gateway audit validity invalid: issued=%s expires=%s now=%s", + attestationView.IssuedAt, attestationView.ExpiresAt, nowProbe) + } + save("05a-gateway-audit.json", map[string]any{"issued_at": attestationView.IssuedAt, "expires_at": attestationView.ExpiresAt, + "now": nowProbe, "validity": attestationView.ExpiresAt.Sub(attestationView.IssuedAt).String(), + "clock_source": "system-clock-unmodified"}) + must(os.WriteFile(filepath.Join(root, "audit.json"), signedRaw, 0600)) + + processes := []map[string]any{} + start := func(binary, label string, values map[string]string) *exec.Cmd { + t.Helper() + cmd := exec.Command(filepath.Join(os.Getenv("KG_R38_BIN"), binary)) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} + for k, v := range values { + cmd.Env = append(cmd.Env, k+"="+v) + } + f, e := os.OpenFile(filepath.Join(root, label+".log"), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) + must(e) + cmd.Stdout = f + cmd.Stderr = f + must(cmd.Start()) + processes = append(processes, map[string]any{"label": label, "pid": cmd.Process.Pid, "binary": binary}) + save("06-processes.json", processes) + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); f.Close() }) + return cmd + } + genv := map[string]string{"OBJECT_GATEWAY_LISTEN": os.Getenv("KG_R38_GATEWAY_ADDR"), "OBJECT_GATEWAY_DATABASE_URL": govDSN, + "OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL": reader.Config().ConnString(), "OBJECT_GATEWAY_ARCHIVE_ROOT": archiveRoot, + "OBJECT_GATEWAY_TARGET_ID": cfg.LibraryTargetIdentity(), "OBJECT_GATEWAY_PROVIDER": "minio", + "OBJECT_GATEWAY_ENDPOINT": env["LIBRARY_PRODUCTION_S3_ENDPOINT"], "OBJECT_GATEWAY_BUCKET": fx.bucket, + "OBJECT_GATEWAY_REGION": "us-east-1", "OBJECT_GATEWAY_ACCESS_KEY": os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"), + "OBJECT_GATEWAY_SECRET_KEY": os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY"), "OBJECT_GATEWAY_DATA_TOKEN": dataToken, + "OBJECT_GATEWAY_CONTROL_TOKEN": controlToken, "OBJECT_GATEWAY_ID": project, + "OBJECT_GATEWAY_AUDIT_FILE": filepath.Join(root, "audit.json"), + "OBJECT_GATEWAY_AUDIT_PUBLIC_KEY": base64.StdEncoding.EncodeToString(pub)} + start("object-gateway", "gateway", genv) + client, e := objectgateway.NewClient(env["LIBRARY_OBJECT_GATEWAY_URL"], dataToken, scope) + must(e) + ready := false + for i := 0; i < 100; i++ { + if client.ReadyForAcquisition(ctx) == nil { + ready = true + break + } + time.Sleep(100 * time.Millisecond) + } + if !ready { + t.Fatal("object gateway not ready; inspect gateway.log") + } + + apiAddr := os.Getenv("KG_R38_API_ADDR") + apiEnv := map[string]string{} + for k, v := range env { + apiEnv[k] = v + } + apiEnv["APP_ADDR"] = apiAddr + apiEnv["GOVERNANCE_SPOOL_ROOT"] = filepath.Join(root, "spool") + apiEnv["RELEASE_WORKER_ENABLED"] = "false" + must(os.MkdirAll(apiEnv["GOVERNANCE_SPOOL_ROOT"], 0700)) + start("api", "api", apiEnv) + apiURL := "http://" + apiAddr + for i := 0; i < 120; i++ { + if response, err := http.Get(apiURL + "/health"); err == nil { + response.Body.Close() + if response.StatusCode == 200 { + break + } + } + if i == 119 { + t.Fatal("api /health never became 200") + } + time.Sleep(200 * time.Millisecond) + } + + // 管理员会话(只改这一次性治理库里的 admin 口令;口令只存在内存里)。 + password := hex.EncodeToString(func() []byte { b := make([]byte, 18); _, _ = rand.Read(b); return b }()) + hash, e := bcrypt.GenerateFromPassword([]byte(password), 10) + must(e) + _, e = gov.Exec(ctx, `UPDATE users SET password_hash=$1, must_change_password=false, failed_attempts=0, + locked_until=NULL, state='active' WHERE username='admin'`, string(hash)) + must(e) + jar, e := cookiejar.New(nil) + must(e) + api := r38API{base: apiURL, client: &http.Client{Jar: jar, Timeout: 30 * time.Second}} + code, body := api.do(t, http.MethodPost, "/api/v1/auth/login", map[string]string{"username": "admin", "password": password}) + if code != http.StatusOK { + t.Fatalf("admin login => %d %v", code, body) + } + if user, _ := body["data"].(map[string]any); user["role"] != "admin" { + t.Fatalf("admin session role = %v", body) + } + save("07-session.json", map[string]any{"login_http": code, "role": "admin"}) + + // ---------- 3) 播种真实发布作业(worker 执行,测试不写目标) ---------- + store := postgresstore.NewStore(gov).WithLibraryTarget(cfg.LibraryTargetIdentity()) + now := time.Now().UTC() + digest, e := governance.LibraryRequestDigest("internal", governance.LibraryJobSync, map[string]any{"target_snapshot_id": source}) + must(e) + must(store.CreateLibraryAuthorization(ctx, governance.LibraryReleaseAuthorization{ID: project + "-auth", Scope: "internal", + SnapshotID: source, ParamsDigest: digest, ExpiresAt: now.Add(time.Hour), CreatedAt: now})) + op, _, e := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{ID: project + "-operation", Scope: "internal", + Kind: governance.LibraryJobSync, TargetSnapshotID: source, RequestDigest: digest, OperationKey: project, CreatedAt: now}) + must(e) + job, _, e := store.CreateLibraryDispatch(ctx, governance.LibraryJob{ID: project + "-job", OperationID: op.ID, Scope: "internal", + Kind: governance.LibraryJobSync, TargetSnapshotID: source, IdempotencyKey: project, RequestDigest: digest}, + governance.SyncStages(), false, now) + must(e) + queueID := job.QueueJobID + if queueID == "" { + must(gov.QueryRow(ctx, "SELECT queue_job_id FROM library_jobs WHERE id=$1", job.ID).Scan(&queueID)) + } + save("08-job.json", map[string]any{"job": job.ID, "operation": job.OperationID, "queue": queueID, + "snapshot": source, "index": fx.index, "bucket": fx.bucket, "target": cfg.LibraryTargetIdentity()}) + + workerEnv := map[string]string{} + for k, v := range env { + workerEnv[k] = v + } + workerEnv["WORKER_ID"] = project + "-worker" + worker := start("worker", "worker", workerEnv) + + waitJob := func(id string, want func(string) bool, seconds int, label string) string { + t.Helper() + deadline := time.Now().Add(time.Duration(seconds) * time.Second) + var state, message string + for time.Now().Before(deadline) { + if e := gov.QueryRow(ctx, "SELECT state,COALESCE(last_error,'') FROM jobs WHERE id=$1", id).Scan(&state, &message); e != nil { + t.Fatalf("read queue %s: %v", id, e) + } + if want(state) { + return state + } + if state == "failed" || state == "completed" || state == "cancelled" || state == "dead" { + if !want(state) { + t.Fatalf("%s terminal state=%s error=%s", label, state, message) + } + } + time.Sleep(200 * time.Millisecond) + } + t.Fatalf("%s timeout (state=%s error=%s)", label, state, message) + return state + } + // 真实发布在四个阶段写完后停在 wiki 之后的屏障上;此时才注入一次性"校验读回不一致", + // 放行后由生产自己的校验判定失败并进入回滚。 + hitPath := filepath.Join(root, "barriers", "after-wiki.hit") + released := false + for i := 0; i < 900; i++ { + if _, e := os.Stat(hitPath); e == nil { + proxy.armVerifyCorruption() + must(os.WriteFile(filepath.Join(root, "barriers", "after-wiki.go"), []byte("release"), 0600)) + save("08a-barrier.json", map[string]any{"point": "after:wiki", "hit": true, "released": true, + "verify_corruption_armed": true, "refuse_clean_dispatch": true}) + released = true + break + } + var state string + must(gov.QueryRow(ctx, "SELECT state FROM jobs WHERE id=$1", queueID).Scan(&state)) + if state == "failed" || state == "completed" || state == "cancelled" || state == "dead" { + t.Fatalf("publish reached %s before the after:wiki barrier", state) + } + time.Sleep(100 * time.Millisecond) + } + if !released { + t.Fatal("publish never reached the after:wiki barrier") + } + if got := waitJob(queueID, func(s string) bool { return s == "failed" }, 180, "publish rollback must fail at the restore boundary"); got != "failed" { + t.Fatalf("publish state = %s", got) + } + libraryAfterPublish, e := store.LibraryJob(ctx, job.ID) + must(e) + save("09-publish-failed.json", libraryAfterPublish) + if libraryAfterPublish.State != governance.LibraryJobFailed { + t.Fatalf("publish library state = %s (%s)", libraryAfterPublish.State, libraryAfterPublish.Error) + } + failureText := libraryAfterPublish.Error + "\n" + libraryAfterPublish.Recovery + if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") { + t.Fatalf("publish failure does not name the refused rollback write: %q", failureText) + } + if string(libraryAfterPublish.Stage) != "rollback" { + t.Fatalf("publish failed at stage %q, want rollback", libraryAfterPublish.Stage) + } + if proxy.corruptionCount() != 1 { + t.Fatalf("verify corruption applied %d times, want exactly 1", proxy.corruptionCount()) + } + + // 回滚失败必须体现为"窗口 held + 两个维护记录 needs_restore",而不是只在作业文案里。 + heldJournal := readR38JSON(t, filepath.Join(root, "maintenance", job.ID+".json")) + save("10-journal-held.json", heldJournal) + if heldJournal["state"] != "held" { + t.Fatalf("freeze journal state = %v, want held", heldJournal["state"]) + } + heldCode, _ := heldJournal["code"].(string) + // 回滚路径的恢复失败由窗口自己记为 restore_blocked(RecoverWindow 的同类失败是 + // restore_failed):两者都表示"目标不可用、内容风险未解除",不是成功收尾。 + if heldCode != "restore_blocked" { + t.Fatalf("freeze journal code = %q, want restore_blocked", heldCode) + } + if heldJournal["restore_blocked"] != true { + t.Fatalf("freeze journal restore_blocked = %v", heldJournal["restore_blocked"]) + } + backupID, _ := heldJournal["backup_snapshot_id"].(string) + if backupID == "" { + t.Fatal("held journal lost the prebackup identity") + } + instance, e := store.InstanceMaintenanceOf(ctx, cfg.LibraryTargetInstance) + must(e) + scopeRow, e := store.LibraryMaintenanceOf(ctx, "internal") + must(e) + if instance == nil || instance.ReleasedAt != nil || instance.State != "needs_restore" || instance.OwnerJobID != job.ID { + t.Fatalf("instance maintenance = %+v", instance) + } + if scopeRow == nil || scopeRow.ReleasedAt != nil || scopeRow.State != "needs_restore" || scopeRow.OwnerJobID != job.ID { + t.Fatalf("scope maintenance = %+v", scopeRow) + } + save("11-maintenance-held.json", map[string]any{"instance": instance, "scope": scopeRow}) + + // 精确边界:restore-finalize-intent 已落盘、marker 原样停在自己世代上、clean 未派发。 + effects := r38IndexEffects(t, archiveRoot, "chunk-target") + save("12-index-effects-held.json", effects) + for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} { + if _, ok := effects[phase]; !ok { + t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects)) + } + } + if _, ok := effects["restored"]; ok { + t.Fatal("held boundary must not carry a restored receipt") + } + heldIndex := recoveryIndexReadback(t, fx) + save("13-index-held.json", heldIndex) + heldDoc, ok := heldIndex["chunk-target"] + if !ok { + t.Fatal("chunk-target missing at held boundary") + } + heldSource := heldDoc["_source"].(map[string]any) + heldContent, _ := heldSource["content"].(string) + if !strings.HasPrefix(heldContent, indexRestoreClaimPrefix) { + t.Fatalf("held index must still carry the restore marker, got %q", heldContent) + } + events, refused := proxy.snapshot() + save("14-wire-held.json", map[string]any{"events": events, "refused": refused}) + if refused != 1 { + t.Fatalf("clean dispatch refusals = %d, want exactly 1", refused) + } + forwarded := 0 + for _, event := range events { + if event["refused"] == true && event["dispatched"] == false { + continue + } + if event["method"] == http.MethodPut && strings.Contains(event["path"].(string), "/_doc/chunk-target") { + forwarded++ + } + } + if forwarded != 0 { + t.Fatalf("refused clean write was forwarded %d times", forwarded) + } + + // HTTP 入口:窗口 held ⇒ 读写入口 409;恢复入口仍是唯一出口。 + code, maintenance := api.do(t, http.MethodGet, "/api/v1/library/maintenance?scope=internal", nil) + if code != http.StatusOK { + t.Fatalf("GET maintenance => %d %v", code, maintenance) + } + save("15-maintenance-http-held.json", map[string]any{"http": code, "body": maintenance}) + if maintenance["held"] != true { + t.Fatalf("maintenance HTTP held = %v", maintenance["held"]) + } + instanceView, _ := maintenance["instance_maintenance"].(map[string]any) + if instanceView == nil || instanceView["state"] != "needs_restore" || instanceView["owner_job_id"] != job.ID { + t.Fatalf("maintenance HTTP instance view = %v", maintenance["instance_maintenance"]) + } + // held 期间入口必须真的关着:连不上,而不是"还有人返回错误页"。 + closedClient := &http.Client{Timeout: 3 * time.Second} + entryOpen := false + entryDetail := "" + if response, err := closedClient.Get(os.Getenv("KG_R38_INGRESS")); err == nil { + entryOpen = response.StatusCode == http.StatusOK + entryDetail = fmt.Sprintf("HTTP %d", response.StatusCode) + response.Body.Close() + } else { + entryDetail = err.Error() + } + save("15a-ingress-held.json", map[string]any{"ingress": os.Getenv("KG_R38_INGRESS"), "open": entryOpen, "detail": entryDetail}) + if entryOpen { + t.Fatalf("target ingress still serves 200 while the window is held: %s", entryDetail) + } + code, blocked := api.do(t, http.MethodPost, "/api/v1/library/restores", map[string]any{"scope": "internal", "snapshot_id": backupID}) + if code != http.StatusConflict || blocked["code"] != "library_target_in_maintenance" { + t.Fatalf("entry while held => %d %v, want 409 library_target_in_maintenance", code, blocked) + } + save("16-entry-409.json", map[string]any{"http": code, "body": blocked}) + code, mismatch := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover", + map[string]any{"scope": "internal", "window_job_id": "some-other-window"}) + if code != http.StatusConflict || mismatch["code"] != "library_window_mismatch" { + t.Fatalf("recover with another window => %d %v", code, mismatch) + } + save("17-window-binding-409.json", map[string]any{"http": code, "body": mismatch}) + + // ---------- 4) 撤故障 → 正常 POST /library/maintenance/recover ---------- + proxy.setRefuseClean(false) + code, accepted := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover", + map[string]any{"scope": "internal", "window_job_id": job.ID}) + if code != http.StatusAccepted { + t.Fatalf("POST recover => %d %v", code, accepted) + } + recoveryJob, _ := accepted["job"].(map[string]any) + recoveryQueueID, _ := recoveryJob["id"].(string) + save("18-recover-accepted.json", accepted) + if accepted["window_job_id"] != job.ID || accepted["scope"] != "internal" { + t.Fatalf("recover response window binding = %v", accepted) + } + if recoveryQueueID != governance.LibraryRecoveryQueueID(job.ID) { + t.Fatalf("recovery queue id = %q", recoveryQueueID) + } + if got := waitJob(recoveryQueueID, func(s string) bool { return s == "completed" }, 300, "recovery must complete"); got != "completed" { + t.Fatalf("recovery state = %s", got) + } + + recoveredJournal := readR38JSON(t, filepath.Join(root, "maintenance", job.ID+".json")) + save("19-journal-recovered.json", recoveredJournal) + if recoveredJournal["state"] != "recovered" { + t.Fatalf("journal after recovery = %v", recoveredJournal["state"]) + } + if recoveredJournal["recovered_backup"] != backupID || recoveredJournal["backup_snapshot_id"] != backupID { + t.Fatalf("recovery bound another backup: %v", recoveredJournal) + } + if digest, _ := recoveredJournal["recovery_digest"].(string); digest == "" { + t.Fatal("recovery finished without a verification digest") + } + instanceAfter, e := store.InstanceMaintenanceOf(ctx, cfg.LibraryTargetInstance) + must(e) + scopeAfter, e := store.LibraryMaintenanceOf(ctx, "internal") + must(e) + if instanceAfter == nil || instanceAfter.ReleasedAt == nil { + t.Fatalf("instance maintenance not released: %+v", instanceAfter) + } + if scopeAfter == nil || scopeAfter.ReleasedAt == nil { + t.Fatalf("scope maintenance not released: %+v", scopeAfter) + } + save("20-maintenance-released.json", map[string]any{"instance": instanceAfter, "scope": scopeAfter}) + code, openMaintenance := api.do(t, http.MethodGet, "/api/v1/library/maintenance?scope=internal", nil) + if code != http.StatusOK || openMaintenance["held"] != false { + t.Fatalf("maintenance after recovery => %d %v", code, openMaintenance) + } + // 入口真的回来了:实际 HTTP 200,而不是"容器起来了"。 + probe := false + for i := 0; i < 100; i++ { + response, err := http.Get(os.Getenv("KG_R38_INGRESS")) + if err == nil { + response.Body.Close() + if response.StatusCode == http.StatusOK { + probe = true + break + } + } + time.Sleep(200 * time.Millisecond) + } + if !probe { + t.Fatal("target ingress did not return HTTP 200 after recovery") + } + save("21-ingress.json", map[string]any{"ingress": os.Getenv("KG_R38_INGRESS"), "http": 200, "open": true, + "maintenance_http": openMaintenance}) + + // ---------- 5) 三个真实存储的全量恢复 ---------- + afterPG := universeDump(t, fx.pool) + save("22-pg-after.json", afterPG) + if diff := diffUniverse(beforePG, afterPG); diff != "" { + t.Fatalf("PG not restored row-by-row: %s", diff) + } + afterObjects := objectInventoryR38(t, ctx, fx) + save("23-objects-after.json", afterObjects) + if !reflect.DeepEqual(objectVersionIDs(beforeObjects), objectVersionIDs(afterObjects)) { + t.Fatalf("object versions differ from before-image:\n before=%v\n after=%v", + objectVersionIDs(beforeObjects), objectVersionIDs(afterObjects)) + } + afterIndex := recoveryIndexReadback(t, fx) + save("24-index-after.json", afterIndex) + if len(afterIndex) != len(beforeIndex) { + t.Fatalf("index document set changed: %d -> %d", len(beforeIndex), len(afterIndex)) + } + for id, old := range beforeIndex { + got, ok := afterIndex[id] + if !ok || !reflect.DeepEqual(old["_source"], got["_source"]) { + t.Fatalf("index document %s not restored to its before-image", id) + } + if id == "chunk-target" && reflect.DeepEqual(old["_seq_no"], got["_seq_no"]) { + t.Fatal("restoration reused the applied generation instead of writing a new one") + } + } + if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil { + t.Fatalf("independent verification after recovery: %v", err) + } + recoveredEffects := r38IndexEffects(t, archiveRoot, "chunk-target") + save("25-index-effects-recovered.json", recoveredEffects) + if _, ok := recoveredEffects["restored"]; !ok { + t.Fatalf("recovery did not persist a restored receipt (have %v)", effectPhases(recoveredEffects)) + } + + // 恢复事实与 receipt 都由真实写入产生,没有人工造 receipt。 + var recoverOps, recoveryVersions int + must(gov.QueryRow(ctx, "SELECT count(*) FROM library_operations WHERE kind='recover'").Scan(&recoverOps)) + must(gov.QueryRow(ctx, "SELECT count(*) FROM library_versions WHERE kind='recovery_verified' AND source_job_id=$1", job.ID).Scan(&recoveryVersions)) + var prebackups int + must(gov.QueryRow(ctx, "SELECT count(*) FROM library_versions WHERE kind='production_prebackup' AND source_job_id=$1 AND snapshot_id=$2", + job.ID, backupID).Scan(&prebackups)) + if recoverOps != 1 || recoveryVersions != 1 || prebackups != 1 { + t.Fatalf("durable recovery identities ops=%d events=%d prebackups=%d", recoverOps, recoveryVersions, prebackups) + } + receipts, deliveries := r38Receipts(t, ctx, gov, fx) + save("26-receipts.json", map[string]any{"receipts": receipts, "deliveries": deliveries}) + if len(receipts) == 0 || len(deliveries) == 0 { + t.Fatal("no gateway receipt/delivery evidence for real object writes") + } + // receipt 必须来自真实网关写入:每条都有对应的 delivery(同计划摘要、同代次), + // 键落在本目标桶的命名空间里;被回滚删掉的那个版本必须有同一网关的 removed 回执。 + committed := 0 + for _, receipt := range receipts { + if receipt.Request.Key == "" || receipt.Version.ID == "" || receipt.Request.Grant.Plan == "" { + t.Fatalf("incomplete receipt %+v", receipt.Request) + } + matched := false + for _, delivery := range deliveries { + if delivery.PlanDigest == receipt.Request.Grant.Plan && delivery.JobID != "" && delivery.AttemptEpoch == receipt.Request.Grant.Epoch { + matched = true + } + } + if !matched { + t.Fatalf("receipt without a durable delivery (plan=%s epoch=%d)", receipt.Request.Grant.Plan, receipt.Request.Grant.Epoch) + } + // 键必须落在本目标桶的对象命名空间里(10001/ 是目标租户前缀)。 + if !strings.HasPrefix(receipt.Request.Key, "10001/") { + t.Fatalf("receipt key %q is outside the isolated target namespace", receipt.Request.Key) + } + if receipt.State != "committed" { + continue + } + committed++ + present := false + for _, version := range afterObjects.Versions { + if aws.ToString(version.Key) == receipt.Request.Key && aws.ToString(version.VersionId) == receipt.Version.ID { + present = true + } + } + if present { + continue + } + removed := false + for _, other := range receipts { + if other.State == "removed" && other.Request.Key == receipt.Request.Key && other.Version.ID == receipt.Version.ID { + removed = true + } + } + if !removed { + t.Fatalf("committed receipt %s@%s vanished without a gateway removal receipt", + receipt.Request.Key, receipt.Version.ID) + } + } + if committed == 0 { + t.Fatal("no committed gateway receipt: object writes cannot be tied to receipt evidence") + } + // Recovery 请求重放:幂等,不得再派发任何写入。 + code, replay := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover", + map[string]any{"scope": "internal", "window_job_id": job.ID}) + if code != http.StatusConflict || replay["code"] != "library_no_held_window" { + t.Fatalf("recover replay on a released window => %d %v", code, replay) + } + save("27-recover-replay.json", map[string]any{"http": code, "body": replay}) + if !reflect.DeepEqual(afterPG, universeDump(t, fx.pool)) { + t.Fatal("replay mutated PG") + } + if !reflect.DeepEqual(afterIndex, recoveryIndexReadback(t, fx)) { + t.Fatal("replay rewrote index generations") + } + + must(worker.Process.Signal(os.Interrupt)) + _ = worker.Wait() + save("28-result.json", map[string]any{"window_job": job.ID, "publish_queue": queueID, + "recovery_queue": recoveryQueueID, "backup": backupID, "scope": "internal", "instance": cfg.LibraryTargetInstance, + "held": map[string]any{"journal": "held/restore_blocked", "maintenance": "needs_restore", "entry_http": 409}, + "recovered": map[string]any{"journal": "recovered", "maintenance_released": true, "entry_http": 200, + "pg_rows_restored": true, "object_versions_restored": true, "es_source_restored": true, + "receipts": len(receipts), "deliveries": len(deliveries), "recover_operations": recoverOps}, + "refusals": refused}) + t.Logf("r38 closed loop: window=%s backup=%s recovery=%s receipts=%d deliveries=%d", + job.ID, backupID, recoveryQueueID, len(receipts), len(deliveries)) +} + +// ---------- 观察辅助 ---------- + +func objectInventoryR38(t *testing.T, ctx context.Context, fx *writeSetFixture) *s3.ListObjectVersionsOutput { + t.Helper() + out, e := fx.objects.s3.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(fx.bucket)}) + if e != nil { + t.Fatalf("object inventory: %v", e) + } + if aws.ToBool(out.IsTruncated) || len(out.DeleteMarkers) != 0 { + t.Fatal("incomplete object inventory or unexpected delete markers") + } + return out +} + +func objectVersionIDs(inventory *s3.ListObjectVersionsOutput) []string { + ids := make([]string, 0, len(inventory.Versions)) + for _, version := range inventory.Versions { + ids = append(ids, aws.ToString(version.Key)+"@"+aws.ToString(version.VersionId)+"#"+aws.ToString(version.ETag)) + } + sort.Strings(ids) + return ids +} + +func readR38JSON(t *testing.T, path string) map[string]any { + t.Helper() + raw, e := os.ReadFile(path) + if e != nil { + t.Fatalf("read %s: %v", path, e) + } + out := map[string]any{} + if e = json.Unmarshal(raw, &out); e != nil { + t.Fatalf("parse %s: %v", path, e) + } + return out +} + +// r38IndexEffects 汇总某个文档在冻结计划下已落盘的阶段证据(phase -> 证据内容)。 +func r38IndexEffects(t *testing.T, archiveRoot, id string) map[string]map[string]any { + t.Helper() + out := map[string]map[string]any{} + dir := filepath.Join(archiveRoot, "effects") + _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil || info == nil || info.IsDir() || !strings.HasSuffix(path, ".json") { + return nil + } + raw, e := os.ReadFile(path) + if e != nil { + return nil + } + var effect struct { + Change map[string]any `json:"change"` + Receipt map[string]any `json:"receipt"` + } + if json.Unmarshal(raw, &effect) != nil || effect.Change == nil { + return nil + } + if got, _ := effect.Change["id"].(string); got != id { + return nil + } + phase := strings.TrimSuffix(filepath.Base(path), ".json") + out[phase] = map[string]any{"change": effect.Change, "receipt": effect.Receipt} + return nil + }) + return out +} + +func effectPhases(effects map[string]map[string]any) []string { + phases := make([]string, 0, len(effects)) + for phase := range effects { + phases = append(phases, phase) + } + sort.Strings(phases) + return phases +} + +func r38Receipts(t *testing.T, ctx context.Context, gov *pgxpool.Pool, fx *writeSetFixture) ([]objectgateway.Receipt, []objectgateway.Claim) { + t.Helper() + rows, e := gov.Query(ctx, "SELECT receipt FROM kg_object_gateway_receipts ORDER BY 1") + if e != nil { + t.Fatalf("receipts: %v", e) + } + receipts := []objectgateway.Receipt{} + for rows.Next() { + var raw []byte + if e := rows.Scan(&raw); e != nil { + rows.Close() + t.Fatalf("scan receipt: %v", e) + } + var receipt objectgateway.Receipt + if e := json.Unmarshal(raw, &receipt); e != nil { + rows.Close() + t.Fatalf("parse receipt: %v", e) + } + receipts = append(receipts, receipt) + } + rows.Close() + deliveryRows, e := gov.Query(ctx, "SELECT claim FROM kg_object_gateway_deliveries ORDER BY attempt_epoch") + if e != nil { + t.Fatalf("deliveries: %v", e) + } + deliveries := []objectgateway.Claim{} + for deliveryRows.Next() { + var raw []byte + if e := deliveryRows.Scan(&raw); e != nil { + deliveryRows.Close() + t.Fatalf("scan delivery: %v", e) + } + var claim objectgateway.Claim + if e := json.Unmarshal(raw, &claim); e != nil { + deliveryRows.Close() + t.Fatalf("parse delivery: %v", e) + } + deliveries = append(deliveries, claim) + } + deliveryRows.Close() + return receipts, deliveries +}