test(release): 正式 worker 与维护恢复入口的 held→恢复闭环验收
build / build (push) Successful in 1m25s

真实 publish 在四类写入完成后由校验不一致失败进入回滚;回滚在
restore-finalize-intent 落盘、marker 仍在自身世代时被拒绝 clean 派发,
窗口 held、两个维护记录 needs_restore、写入口 409;撤故障后
POST /library/maintenance/recover 以同一 window/job/backup 续作成功,
PG 逐行/对象版本/ES _source 全量恢复并重开入口。仅新增验收与运行器,
产品行为与判据未改。
This commit is contained in:
姚章浩
2026-11-02 01:44:03 +08:00
parent 066dc45a25
commit 1d2fff8639
2 changed files with 1008 additions and 0 deletions
+18
View File
@@ -1,5 +1,23 @@
# Changelog
## 开发快照(正式 worker / 维护恢复入口的 held → 恢复闭环验收)
### Added
- 新增正式链路验收 `TestFormalMaintenanceRecoveryHeldThenRecovered`(只在 `KG_R38_RUN` 指定的
一次性隔离部署里运行,常规运行与 CI 跳过):真实 publish 在 database/resources/index/wiki 四类
真实写入完成后,由"校验读回内容与快照不一致"失败进入自动回滚;回滚在
`restore-finalize-intent` 已落盘、marker 仍停在自己世代时被传输层拒绝 clean 派发(拒绝 = 不转发、
不丢响应),窗口保持 held,实例/scope 两个维护记录写 `needs_restore`,写入口返回 409;撤掉一次性
故障后 `POST /api/v1/library/maintenance/recover` 以同一 window/job/backup 续作成功,PG 逐行、
对象版本集合与 ES `_source` 全量恢复,入口重新返回 200。
- 该用例中的对象 receipt 全部由真实对象网关写入产生(committed/removed 两条不可变回执 + 对应
delivery/plan 摘要),没有人工造回执,也没有改核心状态。
### Changed
- 本轮只新增验收与运行器,产品行为、协议与判据均未改变;删除型恢复的 R02-b 缺口保持原样。
## 开发快照(删除型索引恢复不再把 ES 统计计数器当证据)
### Fixed
@@ -0,0 +1,990 @@
package v071
// r38:正式 worker / maintenance / recover API 的"有证据可恢复"闭环。
//
// 场景(全部由真实进程与真实存储产生,测试只播种/观察):
// 1. 真实 publish:worker 执行 apply(database/resources/index 真实写入,对象经对象网关
// 生成 receipt);快照缺 wiki 部件 → wiki 阶段真实失败(与 r37 contractFailWikiTarget 同一定点)。
// 2. 失败后的自动回滚在 restore-finalize-intent 已落盘、marker 仍停在自己世代时被
// 传输层拒绝"clean 派发"(拒绝 = 不转发、不丢响应;ES 里 marker 原样保留)。
// → 回滚失败 → 维护窗口 held + needs_restore(不是 adapter 返回值当结论)。
// 3. HTTP:GET /library/maintenance 看到 held/needs_restore;写入口 POST /library/restores 返回 409。
// 4. 撤一次性故障 → POST /library/maintenance/recover(同一 window/job/backup)→ worker 续作
// 一次 clean 写入 → 校验通过、入口 open、维护释放;PG 逐行/对象版本/ES _source 全量恢复。
//
// 该测试只在 KG_R38_RUN 指定的隔离部署里运行;故障只存在于本进程的 ES 传输层。
import (
"bytes"
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/cookiejar"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"reflect"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/jackc/pgx/v5/pgxpool"
"golang.org/x/crypto/bcrypt"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
"knowledge-governance/internal/objectgateway"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// r38RefusalProxy 是"只在传输层拒绝,不转发"的故障注入:
// 选中的写请求被直接 503,绝不落到真实 ES;其余请求原样转发并记录变更请求现场。
type r38RefusalProxy struct {
url string
mu sync.Mutex
refuseClean bool
corruptVerify bool
events []map[string]any
refused int
corrupted int
root string
}
func newR38RefusalProxy(t *testing.T, root, upstream string) *r38RefusalProxy {
t.Helper()
p := &r38RefusalProxy{root: root, refuseClean: true}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, e := io.ReadAll(r.Body)
if e != nil {
http.Error(w, "read failed", 500)
return
}
p.mu.Lock()
refuseClean := p.refuseClean
p.mu.Unlock()
selected := refuseClean && r.Method == http.MethodPut &&
strings.Contains(r.URL.Path, "/_doc/chunk-target") &&
!bytes.Contains(raw, []byte(indexRestoreClaimPrefix))
if selected {
p.mu.Lock()
p.refused++
p.events = append(p.events, map[string]any{
"method": r.Method, "path": r.URL.RequestURI(), "request": string(raw),
"dispatched": false, "refused": true, "at": time.Now().UTC().Format(time.RFC3339Nano)})
p.flush()
p.mu.Unlock()
// 拒绝发生在派发之前:真实 ES 一次都没有收到这份 clean 写入。
http.Error(w, "r38 injected refusal before clean dispatch", http.StatusServiceUnavailable)
return
}
request, e := http.NewRequestWithContext(r.Context(), r.Method, upstream+r.URL.RequestURI(), bytes.NewReader(raw))
if e != nil {
http.Error(w, "request failed", 500)
return
}
request.Header = r.Header.Clone()
// 读回内容必须是明文 JSON:否则注入的"内容不一致"只会变成无法解析的压缩字节,
// 校验侧会把它当成传输错误而不是真实内容差异。
request.Header.Del("Accept-Encoding")
response, e := http.DefaultClient.Do(request)
if e != nil {
http.Error(w, "upstream failed", 502)
return
}
body, e := io.ReadAll(response.Body)
response.Body.Close()
if e != nil {
http.Error(w, "upstream body failed", 502)
return
}
p.mu.Lock()
corrupt := p.corruptVerify && response.StatusCode == 200 && strings.Contains(r.URL.Path, "_search")
if corrupt {
// 只改这一份读回内容并立刻解除:校验阶段读到"与快照不一致"的真实不一致,
// 后续回滚/恢复的 _mget 与 _doc 请求仍然读到真实 ES。
p.corruptVerify = false
p.corrupted++
body = corruptSearchBody(body)
}
p.mu.Unlock()
if r.Method == http.MethodPut || r.Method == http.MethodPost || r.Method == http.MethodDelete {
p.mu.Lock()
p.events = append(p.events, map[string]any{
"method": r.Method, "path": r.URL.RequestURI(), "request": string(raw),
"status": response.StatusCode, "dispatched": true, "refused": false, "corrupted": corrupt,
"at": time.Now().UTC().Format(time.RFC3339Nano)})
if len(p.events)%8 == 0 {
p.flush()
}
p.mu.Unlock()
}
for k, values := range response.Header {
for _, v := range values {
w.Header().Add(k, v)
}
}
w.WriteHeader(response.StatusCode)
_, _ = w.Write(body)
}))
t.Cleanup(server.Close)
p.url = server.URL
p.mu.Lock()
p.flush()
p.mu.Unlock()
return p
}
func (p *r38RefusalProxy) flush() {
encoded, _ := json.MarshalIndent(p.events, "", " ")
_ = os.WriteFile(filepath.Join(p.root, "es-wire.json"), encoded, 0600)
}
func (p *r38RefusalProxy) setRefuseClean(refuse bool) {
p.mu.Lock()
defer p.mu.Unlock()
p.refuseClean = refuse
p.flush()
}
func (p *r38RefusalProxy) armVerifyCorruption() {
p.mu.Lock()
defer p.mu.Unlock()
p.corruptVerify = true
p.flush()
}
func (p *r38RefusalProxy) corruptionCount() int {
p.mu.Lock()
defer p.mu.Unlock()
return p.corrupted
}
// corruptSearchBody 让校验阶段的读回与快照内容不一致(真实内容差异,不是错误响应)。
func corruptSearchBody(body []byte) []byte {
var payload map[string]any
if json.Unmarshal(body, &payload) != nil {
return body
}
hits, _ := payload["hits"].(map[string]any)["hits"].([]any)
if len(hits) == 0 {
return body
}
hit, _ := hits[0].(map[string]any)
source, _ := hit["_source"].(map[string]any)
if source == nil {
return body
}
content, _ := source["content"].(string)
source["content"] = content + "-r38-verify-mismatch"
encoded, e := json.Marshal(payload)
if e != nil {
return body
}
return encoded
}
func (p *r38RefusalProxy) snapshot() ([]map[string]any, int) {
p.mu.Lock()
defer p.mu.Unlock()
out := append([]map[string]any(nil), p.events...)
return out, p.refused
}
type r38API struct {
base string
client *http.Client
}
func (a r38API) do(t *testing.T, method, path string, payload any) (int, map[string]any) {
t.Helper()
var body []byte
if payload != nil {
encoded, e := json.Marshal(payload)
if e != nil {
t.Fatal(e)
}
body = encoded
}
request, e := http.NewRequest(method, a.base+path, bytes.NewReader(body))
if e != nil {
t.Fatal(e)
}
if body != nil {
request.Header.Set("Content-Type", "application/json")
}
response, e := a.client.Do(request)
if e != nil {
t.Fatalf("%s %s: %v", method, path, e)
}
defer response.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20))
out := map[string]any{}
if len(raw) > 0 {
_ = json.Unmarshal(raw, &out)
}
return response.StatusCode, out
}
func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
root := os.Getenv("KG_R38_RUN")
if root == "" {
t.Skip("requires the dedicated r38 isolated deployment (KG_R38_RUN)")
}
ctx := context.Background()
save := func(name string, value any) {
t.Helper()
raw, e := json.MarshalIndent(value, "", " ")
if e != nil {
t.Fatal(e)
}
if e = os.WriteFile(filepath.Join(root, name), raw, 0600); e != nil {
t.Fatal(e)
}
}
must := func(e error) {
t.Helper()
if e != nil {
t.Fatal(e)
}
}
archiveRoot := filepath.Join(root, "archive")
must(os.MkdirAll(archiveRoot, 0700))
must(os.MkdirAll(filepath.Join(root, "maintenance"), 0700))
// ---------- 1) 隔离目标现场 ----------
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true, archiveRoot: archiveRoot, embeddingDimension: 1024})
seedTargetData(t, fx)
_, e := fx.pool.Exec(ctx, `ALTER TABLE knowledge_bases ADD COLUMN embedding_model_id text, ADD COLUMN vector_store_id text;
CREATE TABLE models(id text PRIMARY KEY,name text,parameters jsonb,deleted_at timestamptz);
CREATE TABLE vector_stores(id text PRIMARY KEY,engine_type text,index_config jsonb,deleted_at timestamptz);
INSERT INTO models VALUES('r38-model','text-embedding-v4','{"embedding_parameters":{"dimension":1024}}',NULL);
INSERT INTO vector_stores VALUES('r38-vector','elasticsearch_v8','{}',NULL);
UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-vector';`)
must(e)
beforePG := universeDump(t, fx.pool)
save("01-pg-before.json", beforePG)
beforeIndex := recoveryIndexReadback(t, fx)
save("02-index-before.json", beforeIndex)
beforeObjects := objectInventoryR38(t, ctx, fx)
save("03-objects-before.json", beforeObjects)
target := fx.target(t)
mapping, e := target.Search().MappingSHA256(ctx)
must(e)
// 快照完整、可封存;发布在四个阶段真实写完之后由"读回内容与快照不一致"的校验失败
// 进入回滚(与 r37 在 wiki 阶段制造失败后回滚同一定点,但这里连 wiki 也是真实写入的)。
source := sealSourceSnapshot(t, fx, nil)
archive, e := librarysnapshot.Open(archiveRoot, 256<<20)
must(e)
manifest, e := archive.Load(source)
must(e)
manifest.Compatibility.EmbeddingDimension = 1024
manifest.Compatibility.IndexMappingSHA256 = mapping
source, e = archive.Seal(manifest)
must(e)
archive.Close()
save("04-snapshot.json", map[string]any{"snapshot": source, "index_mapping_sha256": mapping})
// ---------- 2) 治理库 / 网关 / API / worker ----------
govDSN := os.Getenv("KG_R38_GOVERNANCE_DSN")
gov, e := pgxpool.New(ctx, govDSN)
must(e)
t.Cleanup(gov.Close)
must(postgresstore.ApplyMigrations(ctx, gov))
_, e = gov.Exec(ctx, objectgateway.Schema)
must(e)
reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN)
project := os.Getenv("KG_R38_PROJECT")
proxy := newR38RefusalProxy(t, root, fx.searchBase)
env := map[string]string{
"KG_LIBRARY_TEST_ISOLATED": "1", "DATABASE_URL": govDSN, "LIBRARY_WORKER_ROLE": "release-executor",
"WORKER_POLL_INTERVAL": "200ms", "WORKER_LEASE_DURATION": "30s",
"PRODUCTION_RELEASE_REQUESTS_ENABLED": "true", "PRODUCTION_BACKUP_RESTORE_VERIFIED": "true",
"LIBRARY_ARCHIVE_ROOT": archiveRoot, "LIBRARY_CAPACITY_BUDGET_BYTES": "1073741824",
"LIBRARY_PRODUCTION_DATABASE_URL": os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"),
"LIBRARY_PRODUCTION_ELASTIC_URL": proxy.url, "LIBRARY_PRODUCTION_INDEX": fx.index,
"LIBRARY_PRODUCTION_TENANT_ID": "10001", "LIBRARY_PRODUCTION_KB_INTERNAL": wsTargetKB,
"LIBRARY_PRODUCTION_S3_PROVIDER": "minio", "LIBRARY_PRODUCTION_S3_ENDPOINT": os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT"),
"LIBRARY_PRODUCTION_S3_BUCKET": fx.bucket,
"LIBRARY_PRODUCTION_S3_ACCESS_KEY": os.Getenv("KG_R38_READER_KEY"),
"LIBRARY_PRODUCTION_S3_SECRET_KEY": os.Getenv("KG_R38_READER_SECRET"),
"LIBRARY_TARGET_INSTANCE": project, "LIBRARY_TARGET_PROJECT": project, "LIBRARY_TARGET_WRITERS": "writer",
"LIBRARY_TARGET_NON_WRITERS": "ingress=entry gate,redis=queue,minio=object backend",
"LIBRARY_TARGET_INGRESS_URL": os.Getenv("KG_R38_INGRESS"), "LIBRARY_TARGET_INGRESS_SERVICE": "ingress",
"LIBRARY_TARGET_INGRESS_DRAIN_FILE": filepath.Join(root, "gate/drain.json"),
"LIBRARY_TARGET_REDIS_ADDR": os.Getenv("KG_R38_REDIS_ADDR"), "LIBRARY_TARGET_REDIS_PASSWORD": os.Getenv("KG_R38_REDIS_PASSWORD"),
"LIBRARY_TARGET_REDIS_DB": "0", "LIBRARY_TARGET_JOURNAL_ROOT": filepath.Join(root, "maintenance"),
"LIBRARY_TARGET_DOCKER_SOCKET": "/var/run/docker.sock", "LIBRARY_EMBEDDING_DIMENSION": "1024",
"KG_TEST_BARRIER_DIR": filepath.Join(root, "barriers"), "KG_TEST_BARRIER_POINTS": "after:wiki",
"KG_TEST_BARRIER_TIMEOUT": "300s",
}
env["PRODUCTION_RELEASE_AUTHORIZER_REF"] = project
env["PRODUCTION_RELEASE_AUTH_TOKEN_FILE"] = filepath.Join(root, "authorization-token")
must(os.WriteFile(env["PRODUCTION_RELEASE_AUTH_TOKEN_FILE"], []byte(objectgateway.Digest([]byte(root+"approval"))), 0600))
cfg, e := appconfig.Load(func(k string) string { return env[k] })
must(e)
dataToken := objectgateway.Digest([]byte(root + "data"))
controlToken := objectgateway.Digest([]byte(root + "control"))
env["LIBRARY_OBJECT_GATEWAY_URL"] = "http://" + os.Getenv("KG_R38_GATEWAY_ADDR")
env["LIBRARY_OBJECT_GATEWAY_TOKEN"] = dataToken
// worker 的对象身份是只读的:任何对象写入只能经网关,receipt 因此必然来自真实生产写。
readStore, e := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: env["LIBRARY_PRODUCTION_S3_ENDPOINT"],
Bucket: fx.bucket, AccessKeyID: env["LIBRARY_PRODUCTION_S3_ACCESS_KEY"], SecretAccessKey: env["LIBRARY_PRODUCTION_S3_SECRET_KEY"],
Region: "us-east-1", ForcePathStyle: true})
must(e)
allowedRead, e := readStore.Open(ctx, "minio", wsLocator("backend-target", "minio", fx.bucket, 10001, "res-target"))
must(e)
allowedBody, e := io.ReadAll(allowedRead)
allowedRead.Close()
must(e)
if !bytes.Equal(allowedBody, []byte("12345678")) {
t.Fatal("readonly identity could not read original bytes")
}
if denied := readStore.Put(ctx, "minio", wsLocator("backend-target", "minio", fx.bucket, 10001, "r38-denied"), bytes.NewReader([]byte("denied")), 6); denied == nil {
t.Fatal("worker readonly identity unexpectedly wrote an object")
}
save("05-identity.json", map[string]any{"worker_object_identity": "readonly", "gateway_only_writer": true,
"bucket": fx.bucket, "index": fx.index, "target": cfg.LibraryTargetIdentity(), "project": project,
"readonly_read_bytes": string(allowedBody), "readonly_put_denied": true})
scope := objectgateway.Digest([]byte("minio\x00" + env["LIBRARY_PRODUCTION_S3_ENDPOINT"] + "\x00" + fx.bucket))
pub, priv, e := ed25519.GenerateKey(rand.Reader)
must(e)
digestFile := func(name string) string {
raw, e := os.ReadFile(filepath.Join(root, name))
must(e)
return objectgateway.Digest(raw)
}
initialDrain := filepath.Join(root, "initial-drain.json")
if _, e := os.Stat(initialDrain); e != nil {
t.Fatalf("runner must provide real initial drain evidence: %v", e)
}
payload, e := json.Marshal(objectgateway.Attestation{Protocol: objectgateway.Protocol, Scope: scope, GatewayID: project,
IssuedAt: time.Now().Add(-time.Second), ExpiresAt: time.Now().Add(50 * time.Minute),
WriterInventory: digestFile("05-identity.json"), PolicyEvidence: digestFile("05-identity.json"),
DenialEvidence: digestFile("05-identity.json"), DrainEvidence: digestFile("initial-drain.json")})
must(e)
signedRaw, e := json.Marshal(objectgateway.SignedAttestation{Payload: payload, Signature: ed25519.Sign(priv, payload)})
must(e)
// 审计有效期按真实时钟记录(网关要求不超过 1 小时;这里自己证明没有改系统时间)。
var attestationView objectgateway.Attestation
must(json.Unmarshal(payload, &attestationView))
nowProbe := time.Now()
if attestationView.ExpiresAt.Sub(attestationView.IssuedAt) > time.Hour || nowProbe.After(attestationView.ExpiresAt) {
t.Fatalf("gateway audit validity invalid: issued=%s expires=%s now=%s",
attestationView.IssuedAt, attestationView.ExpiresAt, nowProbe)
}
save("05a-gateway-audit.json", map[string]any{"issued_at": attestationView.IssuedAt, "expires_at": attestationView.ExpiresAt,
"now": nowProbe, "validity": attestationView.ExpiresAt.Sub(attestationView.IssuedAt).String(),
"clock_source": "system-clock-unmodified"})
must(os.WriteFile(filepath.Join(root, "audit.json"), signedRaw, 0600))
processes := []map[string]any{}
start := func(binary, label string, values map[string]string) *exec.Cmd {
t.Helper()
cmd := exec.Command(filepath.Join(os.Getenv("KG_R38_BIN"), binary))
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")}
for k, v := range values {
cmd.Env = append(cmd.Env, k+"="+v)
}
f, e := os.OpenFile(filepath.Join(root, label+".log"), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
must(e)
cmd.Stdout = f
cmd.Stderr = f
must(cmd.Start())
processes = append(processes, map[string]any{"label": label, "pid": cmd.Process.Pid, "binary": binary})
save("06-processes.json", processes)
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); f.Close() })
return cmd
}
genv := map[string]string{"OBJECT_GATEWAY_LISTEN": os.Getenv("KG_R38_GATEWAY_ADDR"), "OBJECT_GATEWAY_DATABASE_URL": govDSN,
"OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL": reader.Config().ConnString(), "OBJECT_GATEWAY_ARCHIVE_ROOT": archiveRoot,
"OBJECT_GATEWAY_TARGET_ID": cfg.LibraryTargetIdentity(), "OBJECT_GATEWAY_PROVIDER": "minio",
"OBJECT_GATEWAY_ENDPOINT": env["LIBRARY_PRODUCTION_S3_ENDPOINT"], "OBJECT_GATEWAY_BUCKET": fx.bucket,
"OBJECT_GATEWAY_REGION": "us-east-1", "OBJECT_GATEWAY_ACCESS_KEY": os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"),
"OBJECT_GATEWAY_SECRET_KEY": os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY"), "OBJECT_GATEWAY_DATA_TOKEN": dataToken,
"OBJECT_GATEWAY_CONTROL_TOKEN": controlToken, "OBJECT_GATEWAY_ID": project,
"OBJECT_GATEWAY_AUDIT_FILE": filepath.Join(root, "audit.json"),
"OBJECT_GATEWAY_AUDIT_PUBLIC_KEY": base64.StdEncoding.EncodeToString(pub)}
start("object-gateway", "gateway", genv)
client, e := objectgateway.NewClient(env["LIBRARY_OBJECT_GATEWAY_URL"], dataToken, scope)
must(e)
ready := false
for i := 0; i < 100; i++ {
if client.ReadyForAcquisition(ctx) == nil {
ready = true
break
}
time.Sleep(100 * time.Millisecond)
}
if !ready {
t.Fatal("object gateway not ready; inspect gateway.log")
}
apiAddr := os.Getenv("KG_R38_API_ADDR")
apiEnv := map[string]string{}
for k, v := range env {
apiEnv[k] = v
}
apiEnv["APP_ADDR"] = apiAddr
apiEnv["GOVERNANCE_SPOOL_ROOT"] = filepath.Join(root, "spool")
apiEnv["RELEASE_WORKER_ENABLED"] = "false"
must(os.MkdirAll(apiEnv["GOVERNANCE_SPOOL_ROOT"], 0700))
start("api", "api", apiEnv)
apiURL := "http://" + apiAddr
for i := 0; i < 120; i++ {
if response, err := http.Get(apiURL + "/health"); err == nil {
response.Body.Close()
if response.StatusCode == 200 {
break
}
}
if i == 119 {
t.Fatal("api /health never became 200")
}
time.Sleep(200 * time.Millisecond)
}
// 管理员会话(只改这一次性治理库里的 admin 口令;口令只存在内存里)。
password := hex.EncodeToString(func() []byte { b := make([]byte, 18); _, _ = rand.Read(b); return b }())
hash, e := bcrypt.GenerateFromPassword([]byte(password), 10)
must(e)
_, e = gov.Exec(ctx, `UPDATE users SET password_hash=$1, must_change_password=false, failed_attempts=0,
locked_until=NULL, state='active' WHERE username='admin'`, string(hash))
must(e)
jar, e := cookiejar.New(nil)
must(e)
api := r38API{base: apiURL, client: &http.Client{Jar: jar, Timeout: 30 * time.Second}}
code, body := api.do(t, http.MethodPost, "/api/v1/auth/login", map[string]string{"username": "admin", "password": password})
if code != http.StatusOK {
t.Fatalf("admin login => %d %v", code, body)
}
if user, _ := body["data"].(map[string]any); user["role"] != "admin" {
t.Fatalf("admin session role = %v", body)
}
save("07-session.json", map[string]any{"login_http": code, "role": "admin"})
// ---------- 3) 播种真实发布作业(worker 执行,测试不写目标) ----------
store := postgresstore.NewStore(gov).WithLibraryTarget(cfg.LibraryTargetIdentity())
now := time.Now().UTC()
digest, e := governance.LibraryRequestDigest("internal", governance.LibraryJobSync, map[string]any{"target_snapshot_id": source})
must(e)
must(store.CreateLibraryAuthorization(ctx, governance.LibraryReleaseAuthorization{ID: project + "-auth", Scope: "internal",
SnapshotID: source, ParamsDigest: digest, ExpiresAt: now.Add(time.Hour), CreatedAt: now}))
op, _, e := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{ID: project + "-operation", Scope: "internal",
Kind: governance.LibraryJobSync, TargetSnapshotID: source, RequestDigest: digest, OperationKey: project, CreatedAt: now})
must(e)
job, _, e := store.CreateLibraryDispatch(ctx, governance.LibraryJob{ID: project + "-job", OperationID: op.ID, Scope: "internal",
Kind: governance.LibraryJobSync, TargetSnapshotID: source, IdempotencyKey: project, RequestDigest: digest},
governance.SyncStages(), false, now)
must(e)
queueID := job.QueueJobID
if queueID == "" {
must(gov.QueryRow(ctx, "SELECT queue_job_id FROM library_jobs WHERE id=$1", job.ID).Scan(&queueID))
}
save("08-job.json", map[string]any{"job": job.ID, "operation": job.OperationID, "queue": queueID,
"snapshot": source, "index": fx.index, "bucket": fx.bucket, "target": cfg.LibraryTargetIdentity()})
workerEnv := map[string]string{}
for k, v := range env {
workerEnv[k] = v
}
workerEnv["WORKER_ID"] = project + "-worker"
worker := start("worker", "worker", workerEnv)
waitJob := func(id string, want func(string) bool, seconds int, label string) string {
t.Helper()
deadline := time.Now().Add(time.Duration(seconds) * time.Second)
var state, message string
for time.Now().Before(deadline) {
if e := gov.QueryRow(ctx, "SELECT state,COALESCE(last_error,'') FROM jobs WHERE id=$1", id).Scan(&state, &message); e != nil {
t.Fatalf("read queue %s: %v", id, e)
}
if want(state) {
return state
}
if state == "failed" || state == "completed" || state == "cancelled" || state == "dead" {
if !want(state) {
t.Fatalf("%s terminal state=%s error=%s", label, state, message)
}
}
time.Sleep(200 * time.Millisecond)
}
t.Fatalf("%s timeout (state=%s error=%s)", label, state, message)
return state
}
// 真实发布在四个阶段写完后停在 wiki 之后的屏障上;此时才注入一次性"校验读回不一致",
// 放行后由生产自己的校验判定失败并进入回滚。
hitPath := filepath.Join(root, "barriers", "after-wiki.hit")
released := false
for i := 0; i < 900; i++ {
if _, e := os.Stat(hitPath); e == nil {
proxy.armVerifyCorruption()
must(os.WriteFile(filepath.Join(root, "barriers", "after-wiki.go"), []byte("release"), 0600))
save("08a-barrier.json", map[string]any{"point": "after:wiki", "hit": true, "released": true,
"verify_corruption_armed": true, "refuse_clean_dispatch": true})
released = true
break
}
var state string
must(gov.QueryRow(ctx, "SELECT state FROM jobs WHERE id=$1", queueID).Scan(&state))
if state == "failed" || state == "completed" || state == "cancelled" || state == "dead" {
t.Fatalf("publish reached %s before the after:wiki barrier", state)
}
time.Sleep(100 * time.Millisecond)
}
if !released {
t.Fatal("publish never reached the after:wiki barrier")
}
if got := waitJob(queueID, func(s string) bool { return s == "failed" }, 180, "publish rollback must fail at the restore boundary"); got != "failed" {
t.Fatalf("publish state = %s", got)
}
libraryAfterPublish, e := store.LibraryJob(ctx, job.ID)
must(e)
save("09-publish-failed.json", libraryAfterPublish)
if libraryAfterPublish.State != governance.LibraryJobFailed {
t.Fatalf("publish library state = %s (%s)", libraryAfterPublish.State, libraryAfterPublish.Error)
}
failureText := libraryAfterPublish.Error + "\n" + libraryAfterPublish.Recovery
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
t.Fatalf("publish failure does not name the refused rollback write: %q", failureText)
}
if string(libraryAfterPublish.Stage) != "rollback" {
t.Fatalf("publish failed at stage %q, want rollback", libraryAfterPublish.Stage)
}
if proxy.corruptionCount() != 1 {
t.Fatalf("verify corruption applied %d times, want exactly 1", proxy.corruptionCount())
}
// 回滚失败必须体现为"窗口 held + 两个维护记录 needs_restore",而不是只在作业文案里。
heldJournal := readR38JSON(t, filepath.Join(root, "maintenance", job.ID+".json"))
save("10-journal-held.json", heldJournal)
if heldJournal["state"] != "held" {
t.Fatalf("freeze journal state = %v, want held", heldJournal["state"])
}
heldCode, _ := heldJournal["code"].(string)
// 回滚路径的恢复失败由窗口自己记为 restore_blocked(RecoverWindow 的同类失败是
// restore_failed):两者都表示"目标不可用、内容风险未解除",不是成功收尾。
if heldCode != "restore_blocked" {
t.Fatalf("freeze journal code = %q, want restore_blocked", heldCode)
}
if heldJournal["restore_blocked"] != true {
t.Fatalf("freeze journal restore_blocked = %v", heldJournal["restore_blocked"])
}
backupID, _ := heldJournal["backup_snapshot_id"].(string)
if backupID == "" {
t.Fatal("held journal lost the prebackup identity")
}
instance, e := store.InstanceMaintenanceOf(ctx, cfg.LibraryTargetInstance)
must(e)
scopeRow, e := store.LibraryMaintenanceOf(ctx, "internal")
must(e)
if instance == nil || instance.ReleasedAt != nil || instance.State != "needs_restore" || instance.OwnerJobID != job.ID {
t.Fatalf("instance maintenance = %+v", instance)
}
if scopeRow == nil || scopeRow.ReleasedAt != nil || scopeRow.State != "needs_restore" || scopeRow.OwnerJobID != job.ID {
t.Fatalf("scope maintenance = %+v", scopeRow)
}
save("11-maintenance-held.json", map[string]any{"instance": instance, "scope": scopeRow})
// 精确边界:restore-finalize-intent 已落盘、marker 原样停在自己世代上、clean 未派发。
effects := r38IndexEffects(t, archiveRoot, "chunk-target")
save("12-index-effects-held.json", effects)
for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} {
if _, ok := effects[phase]; !ok {
t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects))
}
}
if _, ok := effects["restored"]; ok {
t.Fatal("held boundary must not carry a restored receipt")
}
heldIndex := recoveryIndexReadback(t, fx)
save("13-index-held.json", heldIndex)
heldDoc, ok := heldIndex["chunk-target"]
if !ok {
t.Fatal("chunk-target missing at held boundary")
}
heldSource := heldDoc["_source"].(map[string]any)
heldContent, _ := heldSource["content"].(string)
if !strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
t.Fatalf("held index must still carry the restore marker, got %q", heldContent)
}
events, refused := proxy.snapshot()
save("14-wire-held.json", map[string]any{"events": events, "refused": refused})
if refused != 1 {
t.Fatalf("clean dispatch refusals = %d, want exactly 1", refused)
}
forwarded := 0
for _, event := range events {
if event["refused"] == true && event["dispatched"] == false {
continue
}
if event["method"] == http.MethodPut && strings.Contains(event["path"].(string), "/_doc/chunk-target") {
forwarded++
}
}
if forwarded != 0 {
t.Fatalf("refused clean write was forwarded %d times", forwarded)
}
// HTTP 入口:窗口 held ⇒ 读写入口 409;恢复入口仍是唯一出口。
code, maintenance := api.do(t, http.MethodGet, "/api/v1/library/maintenance?scope=internal", nil)
if code != http.StatusOK {
t.Fatalf("GET maintenance => %d %v", code, maintenance)
}
save("15-maintenance-http-held.json", map[string]any{"http": code, "body": maintenance})
if maintenance["held"] != true {
t.Fatalf("maintenance HTTP held = %v", maintenance["held"])
}
instanceView, _ := maintenance["instance_maintenance"].(map[string]any)
if instanceView == nil || instanceView["state"] != "needs_restore" || instanceView["owner_job_id"] != job.ID {
t.Fatalf("maintenance HTTP instance view = %v", maintenance["instance_maintenance"])
}
// held 期间入口必须真的关着:连不上,而不是"还有人返回错误页"。
closedClient := &http.Client{Timeout: 3 * time.Second}
entryOpen := false
entryDetail := ""
if response, err := closedClient.Get(os.Getenv("KG_R38_INGRESS")); err == nil {
entryOpen = response.StatusCode == http.StatusOK
entryDetail = fmt.Sprintf("HTTP %d", response.StatusCode)
response.Body.Close()
} else {
entryDetail = err.Error()
}
save("15a-ingress-held.json", map[string]any{"ingress": os.Getenv("KG_R38_INGRESS"), "open": entryOpen, "detail": entryDetail})
if entryOpen {
t.Fatalf("target ingress still serves 200 while the window is held: %s", entryDetail)
}
code, blocked := api.do(t, http.MethodPost, "/api/v1/library/restores", map[string]any{"scope": "internal", "snapshot_id": backupID})
if code != http.StatusConflict || blocked["code"] != "library_target_in_maintenance" {
t.Fatalf("entry while held => %d %v, want 409 library_target_in_maintenance", code, blocked)
}
save("16-entry-409.json", map[string]any{"http": code, "body": blocked})
code, mismatch := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
map[string]any{"scope": "internal", "window_job_id": "some-other-window"})
if code != http.StatusConflict || mismatch["code"] != "library_window_mismatch" {
t.Fatalf("recover with another window => %d %v", code, mismatch)
}
save("17-window-binding-409.json", map[string]any{"http": code, "body": mismatch})
// ---------- 4) 撤故障 → 正常 POST /library/maintenance/recover ----------
proxy.setRefuseClean(false)
code, accepted := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
map[string]any{"scope": "internal", "window_job_id": job.ID})
if code != http.StatusAccepted {
t.Fatalf("POST recover => %d %v", code, accepted)
}
recoveryJob, _ := accepted["job"].(map[string]any)
recoveryQueueID, _ := recoveryJob["id"].(string)
save("18-recover-accepted.json", accepted)
if accepted["window_job_id"] != job.ID || accepted["scope"] != "internal" {
t.Fatalf("recover response window binding = %v", accepted)
}
if recoveryQueueID != governance.LibraryRecoveryQueueID(job.ID) {
t.Fatalf("recovery queue id = %q", recoveryQueueID)
}
if got := waitJob(recoveryQueueID, func(s string) bool { return s == "completed" }, 300, "recovery must complete"); got != "completed" {
t.Fatalf("recovery state = %s", got)
}
recoveredJournal := readR38JSON(t, filepath.Join(root, "maintenance", job.ID+".json"))
save("19-journal-recovered.json", recoveredJournal)
if recoveredJournal["state"] != "recovered" {
t.Fatalf("journal after recovery = %v", recoveredJournal["state"])
}
if recoveredJournal["recovered_backup"] != backupID || recoveredJournal["backup_snapshot_id"] != backupID {
t.Fatalf("recovery bound another backup: %v", recoveredJournal)
}
if digest, _ := recoveredJournal["recovery_digest"].(string); digest == "" {
t.Fatal("recovery finished without a verification digest")
}
instanceAfter, e := store.InstanceMaintenanceOf(ctx, cfg.LibraryTargetInstance)
must(e)
scopeAfter, e := store.LibraryMaintenanceOf(ctx, "internal")
must(e)
if instanceAfter == nil || instanceAfter.ReleasedAt == nil {
t.Fatalf("instance maintenance not released: %+v", instanceAfter)
}
if scopeAfter == nil || scopeAfter.ReleasedAt == nil {
t.Fatalf("scope maintenance not released: %+v", scopeAfter)
}
save("20-maintenance-released.json", map[string]any{"instance": instanceAfter, "scope": scopeAfter})
code, openMaintenance := api.do(t, http.MethodGet, "/api/v1/library/maintenance?scope=internal", nil)
if code != http.StatusOK || openMaintenance["held"] != false {
t.Fatalf("maintenance after recovery => %d %v", code, openMaintenance)
}
// 入口真的回来了:实际 HTTP 200,而不是"容器起来了"。
probe := false
for i := 0; i < 100; i++ {
response, err := http.Get(os.Getenv("KG_R38_INGRESS"))
if err == nil {
response.Body.Close()
if response.StatusCode == http.StatusOK {
probe = true
break
}
}
time.Sleep(200 * time.Millisecond)
}
if !probe {
t.Fatal("target ingress did not return HTTP 200 after recovery")
}
save("21-ingress.json", map[string]any{"ingress": os.Getenv("KG_R38_INGRESS"), "http": 200, "open": true,
"maintenance_http": openMaintenance})
// ---------- 5) 三个真实存储的全量恢复 ----------
afterPG := universeDump(t, fx.pool)
save("22-pg-after.json", afterPG)
if diff := diffUniverse(beforePG, afterPG); diff != "" {
t.Fatalf("PG not restored row-by-row: %s", diff)
}
afterObjects := objectInventoryR38(t, ctx, fx)
save("23-objects-after.json", afterObjects)
if !reflect.DeepEqual(objectVersionIDs(beforeObjects), objectVersionIDs(afterObjects)) {
t.Fatalf("object versions differ from before-image:\n before=%v\n after=%v",
objectVersionIDs(beforeObjects), objectVersionIDs(afterObjects))
}
afterIndex := recoveryIndexReadback(t, fx)
save("24-index-after.json", afterIndex)
if len(afterIndex) != len(beforeIndex) {
t.Fatalf("index document set changed: %d -> %d", len(beforeIndex), len(afterIndex))
}
for id, old := range beforeIndex {
got, ok := afterIndex[id]
if !ok || !reflect.DeepEqual(old["_source"], got["_source"]) {
t.Fatalf("index document %s not restored to its before-image", id)
}
if id == "chunk-target" && reflect.DeepEqual(old["_seq_no"], got["_seq_no"]) {
t.Fatal("restoration reused the applied generation instead of writing a new one")
}
}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
t.Fatalf("independent verification after recovery: %v", err)
}
recoveredEffects := r38IndexEffects(t, archiveRoot, "chunk-target")
save("25-index-effects-recovered.json", recoveredEffects)
if _, ok := recoveredEffects["restored"]; !ok {
t.Fatalf("recovery did not persist a restored receipt (have %v)", effectPhases(recoveredEffects))
}
// 恢复事实与 receipt 都由真实写入产生,没有人工造 receipt。
var recoverOps, recoveryVersions int
must(gov.QueryRow(ctx, "SELECT count(*) FROM library_operations WHERE kind='recover'").Scan(&recoverOps))
must(gov.QueryRow(ctx, "SELECT count(*) FROM library_versions WHERE kind='recovery_verified' AND source_job_id=$1", job.ID).Scan(&recoveryVersions))
var prebackups int
must(gov.QueryRow(ctx, "SELECT count(*) FROM library_versions WHERE kind='production_prebackup' AND source_job_id=$1 AND snapshot_id=$2",
job.ID, backupID).Scan(&prebackups))
if recoverOps != 1 || recoveryVersions != 1 || prebackups != 1 {
t.Fatalf("durable recovery identities ops=%d events=%d prebackups=%d", recoverOps, recoveryVersions, prebackups)
}
receipts, deliveries := r38Receipts(t, ctx, gov, fx)
save("26-receipts.json", map[string]any{"receipts": receipts, "deliveries": deliveries})
if len(receipts) == 0 || len(deliveries) == 0 {
t.Fatal("no gateway receipt/delivery evidence for real object writes")
}
// receipt 必须来自真实网关写入:每条都有对应的 delivery(同计划摘要、同代次),
// 键落在本目标桶的命名空间里;被回滚删掉的那个版本必须有同一网关的 removed 回执。
committed := 0
for _, receipt := range receipts {
if receipt.Request.Key == "" || receipt.Version.ID == "" || receipt.Request.Grant.Plan == "" {
t.Fatalf("incomplete receipt %+v", receipt.Request)
}
matched := false
for _, delivery := range deliveries {
if delivery.PlanDigest == receipt.Request.Grant.Plan && delivery.JobID != "" && delivery.AttemptEpoch == receipt.Request.Grant.Epoch {
matched = true
}
}
if !matched {
t.Fatalf("receipt without a durable delivery (plan=%s epoch=%d)", receipt.Request.Grant.Plan, receipt.Request.Grant.Epoch)
}
// 键必须落在本目标桶的对象命名空间里(10001/ 是目标租户前缀)。
if !strings.HasPrefix(receipt.Request.Key, "10001/") {
t.Fatalf("receipt key %q is outside the isolated target namespace", receipt.Request.Key)
}
if receipt.State != "committed" {
continue
}
committed++
present := false
for _, version := range afterObjects.Versions {
if aws.ToString(version.Key) == receipt.Request.Key && aws.ToString(version.VersionId) == receipt.Version.ID {
present = true
}
}
if present {
continue
}
removed := false
for _, other := range receipts {
if other.State == "removed" && other.Request.Key == receipt.Request.Key && other.Version.ID == receipt.Version.ID {
removed = true
}
}
if !removed {
t.Fatalf("committed receipt %s@%s vanished without a gateway removal receipt",
receipt.Request.Key, receipt.Version.ID)
}
}
if committed == 0 {
t.Fatal("no committed gateway receipt: object writes cannot be tied to receipt evidence")
}
// Recovery 请求重放:幂等,不得再派发任何写入。
code, replay := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
map[string]any{"scope": "internal", "window_job_id": job.ID})
if code != http.StatusConflict || replay["code"] != "library_no_held_window" {
t.Fatalf("recover replay on a released window => %d %v", code, replay)
}
save("27-recover-replay.json", map[string]any{"http": code, "body": replay})
if !reflect.DeepEqual(afterPG, universeDump(t, fx.pool)) {
t.Fatal("replay mutated PG")
}
if !reflect.DeepEqual(afterIndex, recoveryIndexReadback(t, fx)) {
t.Fatal("replay rewrote index generations")
}
must(worker.Process.Signal(os.Interrupt))
_ = worker.Wait()
save("28-result.json", map[string]any{"window_job": job.ID, "publish_queue": queueID,
"recovery_queue": recoveryQueueID, "backup": backupID, "scope": "internal", "instance": cfg.LibraryTargetInstance,
"held": map[string]any{"journal": "held/restore_blocked", "maintenance": "needs_restore", "entry_http": 409},
"recovered": map[string]any{"journal": "recovered", "maintenance_released": true, "entry_http": 200,
"pg_rows_restored": true, "object_versions_restored": true, "es_source_restored": true,
"receipts": len(receipts), "deliveries": len(deliveries), "recover_operations": recoverOps},
"refusals": refused})
t.Logf("r38 closed loop: window=%s backup=%s recovery=%s receipts=%d deliveries=%d",
job.ID, backupID, recoveryQueueID, len(receipts), len(deliveries))
}
// ---------- 观察辅助 ----------
func objectInventoryR38(t *testing.T, ctx context.Context, fx *writeSetFixture) *s3.ListObjectVersionsOutput {
t.Helper()
out, e := fx.objects.s3.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(fx.bucket)})
if e != nil {
t.Fatalf("object inventory: %v", e)
}
if aws.ToBool(out.IsTruncated) || len(out.DeleteMarkers) != 0 {
t.Fatal("incomplete object inventory or unexpected delete markers")
}
return out
}
func objectVersionIDs(inventory *s3.ListObjectVersionsOutput) []string {
ids := make([]string, 0, len(inventory.Versions))
for _, version := range inventory.Versions {
ids = append(ids, aws.ToString(version.Key)+"@"+aws.ToString(version.VersionId)+"#"+aws.ToString(version.ETag))
}
sort.Strings(ids)
return ids
}
func readR38JSON(t *testing.T, path string) map[string]any {
t.Helper()
raw, e := os.ReadFile(path)
if e != nil {
t.Fatalf("read %s: %v", path, e)
}
out := map[string]any{}
if e = json.Unmarshal(raw, &out); e != nil {
t.Fatalf("parse %s: %v", path, e)
}
return out
}
// r38IndexEffects 汇总某个文档在冻结计划下已落盘的阶段证据(phase -> 证据内容)。
func r38IndexEffects(t *testing.T, archiveRoot, id string) map[string]map[string]any {
t.Helper()
out := map[string]map[string]any{}
dir := filepath.Join(archiveRoot, "effects")
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil || info == nil || info.IsDir() || !strings.HasSuffix(path, ".json") {
return nil
}
raw, e := os.ReadFile(path)
if e != nil {
return nil
}
var effect struct {
Change map[string]any `json:"change"`
Receipt map[string]any `json:"receipt"`
}
if json.Unmarshal(raw, &effect) != nil || effect.Change == nil {
return nil
}
if got, _ := effect.Change["id"].(string); got != id {
return nil
}
phase := strings.TrimSuffix(filepath.Base(path), ".json")
out[phase] = map[string]any{"change": effect.Change, "receipt": effect.Receipt}
return nil
})
return out
}
func effectPhases(effects map[string]map[string]any) []string {
phases := make([]string, 0, len(effects))
for phase := range effects {
phases = append(phases, phase)
}
sort.Strings(phases)
return phases
}
func r38Receipts(t *testing.T, ctx context.Context, gov *pgxpool.Pool, fx *writeSetFixture) ([]objectgateway.Receipt, []objectgateway.Claim) {
t.Helper()
rows, e := gov.Query(ctx, "SELECT receipt FROM kg_object_gateway_receipts ORDER BY 1")
if e != nil {
t.Fatalf("receipts: %v", e)
}
receipts := []objectgateway.Receipt{}
for rows.Next() {
var raw []byte
if e := rows.Scan(&raw); e != nil {
rows.Close()
t.Fatalf("scan receipt: %v", e)
}
var receipt objectgateway.Receipt
if e := json.Unmarshal(raw, &receipt); e != nil {
rows.Close()
t.Fatalf("parse receipt: %v", e)
}
receipts = append(receipts, receipt)
}
rows.Close()
deliveryRows, e := gov.Query(ctx, "SELECT claim FROM kg_object_gateway_deliveries ORDER BY attempt_epoch")
if e != nil {
t.Fatalf("deliveries: %v", e)
}
deliveries := []objectgateway.Claim{}
for deliveryRows.Next() {
var raw []byte
if e := deliveryRows.Scan(&raw); e != nil {
deliveryRows.Close()
t.Fatalf("scan delivery: %v", e)
}
var claim objectgateway.Claim
if e := json.Unmarshal(raw, &claim); e != nil {
deliveryRows.Close()
t.Fatalf("parse delivery: %v", e)
}
deliveries = append(deliveries, claim)
}
deliveryRows.Close()
return receipts, deliveries
}