真实 publish 在四类写入完成后由校验不一致失败进入回滚;回滚在 restore-finalize-intent 落盘、marker 仍在自身世代时被拒绝 clean 派发, 窗口 held、两个维护记录 needs_restore、写入口 409;撤故障后 POST /library/maintenance/recover 以同一 window/job/backup 续作成功, PG 逐行/对象版本/ES _source 全量恢复并重开入口。仅新增验收与运行器, 产品行为与判据未改。
This commit is contained in:
@@ -1,5 +1,23 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照(正式 worker / 维护恢复入口的 held → 恢复闭环验收)
|
||||
|
||||
### Added
|
||||
|
||||
- 新增正式链路验收 `TestFormalMaintenanceRecoveryHeldThenRecovered`(只在 `KG_R38_RUN` 指定的
|
||||
一次性隔离部署里运行,常规运行与 CI 跳过):真实 publish 在 database/resources/index/wiki 四类
|
||||
真实写入完成后,由"校验读回内容与快照不一致"失败进入自动回滚;回滚在
|
||||
`restore-finalize-intent` 已落盘、marker 仍停在自己世代时被传输层拒绝 clean 派发(拒绝 = 不转发、
|
||||
不丢响应),窗口保持 held,实例/scope 两个维护记录写 `needs_restore`,写入口返回 409;撤掉一次性
|
||||
故障后 `POST /api/v1/library/maintenance/recover` 以同一 window/job/backup 续作成功,PG 逐行、
|
||||
对象版本集合与 ES `_source` 全量恢复,入口重新返回 200。
|
||||
- 该用例中的对象 receipt 全部由真实对象网关写入产生(committed/removed 两条不可变回执 + 对应
|
||||
delivery/plan 摘要),没有人工造回执,也没有改核心状态。
|
||||
|
||||
### Changed
|
||||
|
||||
- 本轮只新增验收与运行器,产品行为、协议与判据均未改变;删除型恢复的 R02-b 缺口保持原样。
|
||||
|
||||
## 开发快照(删除型索引恢复不再把 ES 统计计数器当证据)
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -0,0 +1,990 @@
|
||||
package v071
|
||||
|
||||
// r38:正式 worker / maintenance / recover API 的"有证据可恢复"闭环。
|
||||
//
|
||||
// 场景(全部由真实进程与真实存储产生,测试只播种/观察):
|
||||
// 1. 真实 publish:worker 执行 apply(database/resources/index 真实写入,对象经对象网关
|
||||
// 生成 receipt);快照缺 wiki 部件 → wiki 阶段真实失败(与 r37 contractFailWikiTarget 同一定点)。
|
||||
// 2. 失败后的自动回滚在 restore-finalize-intent 已落盘、marker 仍停在自己世代时被
|
||||
// 传输层拒绝"clean 派发"(拒绝 = 不转发、不丢响应;ES 里 marker 原样保留)。
|
||||
// → 回滚失败 → 维护窗口 held + needs_restore(不是 adapter 返回值当结论)。
|
||||
// 3. HTTP:GET /library/maintenance 看到 held/needs_restore;写入口 POST /library/restores 返回 409。
|
||||
// 4. 撤一次性故障 → POST /library/maintenance/recover(同一 window/job/backup)→ worker 续作
|
||||
// 一次 clean 写入 → 校验通过、入口 open、维护释放;PG 逐行/对象版本/ES _source 全量恢复。
|
||||
//
|
||||
// 该测试只在 KG_R38_RUN 指定的隔离部署里运行;故障只存在于本进程的 ES 传输层。
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
// r38RefusalProxy 是"只在传输层拒绝,不转发"的故障注入:
|
||||
// 选中的写请求被直接 503,绝不落到真实 ES;其余请求原样转发并记录变更请求现场。
|
||||
type r38RefusalProxy struct {
|
||||
url string
|
||||
mu sync.Mutex
|
||||
refuseClean bool
|
||||
corruptVerify bool
|
||||
events []map[string]any
|
||||
refused int
|
||||
corrupted int
|
||||
root string
|
||||
}
|
||||
|
||||
func newR38RefusalProxy(t *testing.T, root, upstream string) *r38RefusalProxy {
|
||||
t.Helper()
|
||||
p := &r38RefusalProxy{root: root, refuseClean: true}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
raw, e := io.ReadAll(r.Body)
|
||||
if e != nil {
|
||||
http.Error(w, "read failed", 500)
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
refuseClean := p.refuseClean
|
||||
p.mu.Unlock()
|
||||
selected := refuseClean && r.Method == http.MethodPut &&
|
||||
strings.Contains(r.URL.Path, "/_doc/chunk-target") &&
|
||||
!bytes.Contains(raw, []byte(indexRestoreClaimPrefix))
|
||||
if selected {
|
||||
p.mu.Lock()
|
||||
p.refused++
|
||||
p.events = append(p.events, map[string]any{
|
||||
"method": r.Method, "path": r.URL.RequestURI(), "request": string(raw),
|
||||
"dispatched": false, "refused": true, "at": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
p.flush()
|
||||
p.mu.Unlock()
|
||||
// 拒绝发生在派发之前:真实 ES 一次都没有收到这份 clean 写入。
|
||||
http.Error(w, "r38 injected refusal before clean dispatch", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
request, e := http.NewRequestWithContext(r.Context(), r.Method, upstream+r.URL.RequestURI(), bytes.NewReader(raw))
|
||||
if e != nil {
|
||||
http.Error(w, "request failed", 500)
|
||||
return
|
||||
}
|
||||
request.Header = r.Header.Clone()
|
||||
// 读回内容必须是明文 JSON:否则注入的"内容不一致"只会变成无法解析的压缩字节,
|
||||
// 校验侧会把它当成传输错误而不是真实内容差异。
|
||||
request.Header.Del("Accept-Encoding")
|
||||
response, e := http.DefaultClient.Do(request)
|
||||
if e != nil {
|
||||
http.Error(w, "upstream failed", 502)
|
||||
return
|
||||
}
|
||||
body, e := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if e != nil {
|
||||
http.Error(w, "upstream body failed", 502)
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
corrupt := p.corruptVerify && response.StatusCode == 200 && strings.Contains(r.URL.Path, "_search")
|
||||
if corrupt {
|
||||
// 只改这一份读回内容并立刻解除:校验阶段读到"与快照不一致"的真实不一致,
|
||||
// 后续回滚/恢复的 _mget 与 _doc 请求仍然读到真实 ES。
|
||||
p.corruptVerify = false
|
||||
p.corrupted++
|
||||
body = corruptSearchBody(body)
|
||||
}
|
||||
p.mu.Unlock()
|
||||
if r.Method == http.MethodPut || r.Method == http.MethodPost || r.Method == http.MethodDelete {
|
||||
p.mu.Lock()
|
||||
p.events = append(p.events, map[string]any{
|
||||
"method": r.Method, "path": r.URL.RequestURI(), "request": string(raw),
|
||||
"status": response.StatusCode, "dispatched": true, "refused": false, "corrupted": corrupt,
|
||||
"at": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
if len(p.events)%8 == 0 {
|
||||
p.flush()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
for k, values := range response.Header {
|
||||
for _, v := range values {
|
||||
w.Header().Add(k, v)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(response.StatusCode)
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
p.url = server.URL
|
||||
p.mu.Lock()
|
||||
p.flush()
|
||||
p.mu.Unlock()
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *r38RefusalProxy) flush() {
|
||||
encoded, _ := json.MarshalIndent(p.events, "", " ")
|
||||
_ = os.WriteFile(filepath.Join(p.root, "es-wire.json"), encoded, 0600)
|
||||
}
|
||||
|
||||
func (p *r38RefusalProxy) setRefuseClean(refuse bool) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.refuseClean = refuse
|
||||
p.flush()
|
||||
}
|
||||
|
||||
func (p *r38RefusalProxy) armVerifyCorruption() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.corruptVerify = true
|
||||
p.flush()
|
||||
}
|
||||
|
||||
func (p *r38RefusalProxy) corruptionCount() int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.corrupted
|
||||
}
|
||||
|
||||
// corruptSearchBody 让校验阶段的读回与快照内容不一致(真实内容差异,不是错误响应)。
|
||||
func corruptSearchBody(body []byte) []byte {
|
||||
var payload map[string]any
|
||||
if json.Unmarshal(body, &payload) != nil {
|
||||
return body
|
||||
}
|
||||
hits, _ := payload["hits"].(map[string]any)["hits"].([]any)
|
||||
if len(hits) == 0 {
|
||||
return body
|
||||
}
|
||||
hit, _ := hits[0].(map[string]any)
|
||||
source, _ := hit["_source"].(map[string]any)
|
||||
if source == nil {
|
||||
return body
|
||||
}
|
||||
content, _ := source["content"].(string)
|
||||
source["content"] = content + "-r38-verify-mismatch"
|
||||
encoded, e := json.Marshal(payload)
|
||||
if e != nil {
|
||||
return body
|
||||
}
|
||||
return encoded
|
||||
}
|
||||
|
||||
func (p *r38RefusalProxy) snapshot() ([]map[string]any, int) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := append([]map[string]any(nil), p.events...)
|
||||
return out, p.refused
|
||||
}
|
||||
|
||||
type r38API struct {
|
||||
base string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (a r38API) do(t *testing.T, method, path string, payload any) (int, map[string]any) {
|
||||
t.Helper()
|
||||
var body []byte
|
||||
if payload != nil {
|
||||
encoded, e := json.Marshal(payload)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
body = encoded
|
||||
}
|
||||
request, e := http.NewRequest(method, a.base+path, bytes.NewReader(body))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if body != nil {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
response, e := a.client.Do(request)
|
||||
if e != nil {
|
||||
t.Fatalf("%s %s: %v", method, path, e)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
raw, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20))
|
||||
out := map[string]any{}
|
||||
if len(raw) > 0 {
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
}
|
||||
return response.StatusCode, out
|
||||
}
|
||||
|
||||
func TestFormalMaintenanceRecoveryHeldThenRecovered(t *testing.T) {
|
||||
root := os.Getenv("KG_R38_RUN")
|
||||
if root == "" {
|
||||
t.Skip("requires the dedicated r38 isolated deployment (KG_R38_RUN)")
|
||||
}
|
||||
ctx := context.Background()
|
||||
save := func(name string, value any) {
|
||||
t.Helper()
|
||||
raw, e := json.MarshalIndent(value, "", " ")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = os.WriteFile(filepath.Join(root, name), raw, 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
must := func(e error) {
|
||||
t.Helper()
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
archiveRoot := filepath.Join(root, "archive")
|
||||
must(os.MkdirAll(archiveRoot, 0700))
|
||||
must(os.MkdirAll(filepath.Join(root, "maintenance"), 0700))
|
||||
|
||||
// ---------- 1) 隔离目标现场 ----------
|
||||
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true, archiveRoot: archiveRoot, embeddingDimension: 1024})
|
||||
seedTargetData(t, fx)
|
||||
_, e := fx.pool.Exec(ctx, `ALTER TABLE knowledge_bases ADD COLUMN embedding_model_id text, ADD COLUMN vector_store_id text;
|
||||
CREATE TABLE models(id text PRIMARY KEY,name text,parameters jsonb,deleted_at timestamptz);
|
||||
CREATE TABLE vector_stores(id text PRIMARY KEY,engine_type text,index_config jsonb,deleted_at timestamptz);
|
||||
INSERT INTO models VALUES('r38-model','text-embedding-v4','{"embedding_parameters":{"dimension":1024}}',NULL);
|
||||
INSERT INTO vector_stores VALUES('r38-vector','elasticsearch_v8','{}',NULL);
|
||||
UPDATE knowledge_bases SET embedding_model_id='r38-model',vector_store_id='r38-vector';`)
|
||||
must(e)
|
||||
beforePG := universeDump(t, fx.pool)
|
||||
save("01-pg-before.json", beforePG)
|
||||
beforeIndex := recoveryIndexReadback(t, fx)
|
||||
save("02-index-before.json", beforeIndex)
|
||||
beforeObjects := objectInventoryR38(t, ctx, fx)
|
||||
save("03-objects-before.json", beforeObjects)
|
||||
|
||||
target := fx.target(t)
|
||||
mapping, e := target.Search().MappingSHA256(ctx)
|
||||
must(e)
|
||||
// 快照完整、可封存;发布在四个阶段真实写完之后由"读回内容与快照不一致"的校验失败
|
||||
// 进入回滚(与 r37 在 wiki 阶段制造失败后回滚同一定点,但这里连 wiki 也是真实写入的)。
|
||||
source := sealSourceSnapshot(t, fx, nil)
|
||||
archive, e := librarysnapshot.Open(archiveRoot, 256<<20)
|
||||
must(e)
|
||||
manifest, e := archive.Load(source)
|
||||
must(e)
|
||||
manifest.Compatibility.EmbeddingDimension = 1024
|
||||
manifest.Compatibility.IndexMappingSHA256 = mapping
|
||||
source, e = archive.Seal(manifest)
|
||||
must(e)
|
||||
archive.Close()
|
||||
save("04-snapshot.json", map[string]any{"snapshot": source, "index_mapping_sha256": mapping})
|
||||
|
||||
// ---------- 2) 治理库 / 网关 / API / worker ----------
|
||||
govDSN := os.Getenv("KG_R38_GOVERNANCE_DSN")
|
||||
gov, e := pgxpool.New(ctx, govDSN)
|
||||
must(e)
|
||||
t.Cleanup(gov.Close)
|
||||
must(postgresstore.ApplyMigrations(ctx, gov))
|
||||
_, e = gov.Exec(ctx, objectgateway.Schema)
|
||||
must(e)
|
||||
reader := gatewayReadOnlyGovernance(t, ctx, gov, govDSN)
|
||||
project := os.Getenv("KG_R38_PROJECT")
|
||||
proxy := newR38RefusalProxy(t, root, fx.searchBase)
|
||||
|
||||
env := map[string]string{
|
||||
"KG_LIBRARY_TEST_ISOLATED": "1", "DATABASE_URL": govDSN, "LIBRARY_WORKER_ROLE": "release-executor",
|
||||
"WORKER_POLL_INTERVAL": "200ms", "WORKER_LEASE_DURATION": "30s",
|
||||
"PRODUCTION_RELEASE_REQUESTS_ENABLED": "true", "PRODUCTION_BACKUP_RESTORE_VERIFIED": "true",
|
||||
"LIBRARY_ARCHIVE_ROOT": archiveRoot, "LIBRARY_CAPACITY_BUDGET_BYTES": "1073741824",
|
||||
"LIBRARY_PRODUCTION_DATABASE_URL": os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"),
|
||||
"LIBRARY_PRODUCTION_ELASTIC_URL": proxy.url, "LIBRARY_PRODUCTION_INDEX": fx.index,
|
||||
"LIBRARY_PRODUCTION_TENANT_ID": "10001", "LIBRARY_PRODUCTION_KB_INTERNAL": wsTargetKB,
|
||||
"LIBRARY_PRODUCTION_S3_PROVIDER": "minio", "LIBRARY_PRODUCTION_S3_ENDPOINT": os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT"),
|
||||
"LIBRARY_PRODUCTION_S3_BUCKET": fx.bucket,
|
||||
"LIBRARY_PRODUCTION_S3_ACCESS_KEY": os.Getenv("KG_R38_READER_KEY"),
|
||||
"LIBRARY_PRODUCTION_S3_SECRET_KEY": os.Getenv("KG_R38_READER_SECRET"),
|
||||
"LIBRARY_TARGET_INSTANCE": project, "LIBRARY_TARGET_PROJECT": project, "LIBRARY_TARGET_WRITERS": "writer",
|
||||
"LIBRARY_TARGET_NON_WRITERS": "ingress=entry gate,redis=queue,minio=object backend",
|
||||
"LIBRARY_TARGET_INGRESS_URL": os.Getenv("KG_R38_INGRESS"), "LIBRARY_TARGET_INGRESS_SERVICE": "ingress",
|
||||
"LIBRARY_TARGET_INGRESS_DRAIN_FILE": filepath.Join(root, "gate/drain.json"),
|
||||
"LIBRARY_TARGET_REDIS_ADDR": os.Getenv("KG_R38_REDIS_ADDR"), "LIBRARY_TARGET_REDIS_PASSWORD": os.Getenv("KG_R38_REDIS_PASSWORD"),
|
||||
"LIBRARY_TARGET_REDIS_DB": "0", "LIBRARY_TARGET_JOURNAL_ROOT": filepath.Join(root, "maintenance"),
|
||||
"LIBRARY_TARGET_DOCKER_SOCKET": "/var/run/docker.sock", "LIBRARY_EMBEDDING_DIMENSION": "1024",
|
||||
"KG_TEST_BARRIER_DIR": filepath.Join(root, "barriers"), "KG_TEST_BARRIER_POINTS": "after:wiki",
|
||||
"KG_TEST_BARRIER_TIMEOUT": "300s",
|
||||
}
|
||||
env["PRODUCTION_RELEASE_AUTHORIZER_REF"] = project
|
||||
env["PRODUCTION_RELEASE_AUTH_TOKEN_FILE"] = filepath.Join(root, "authorization-token")
|
||||
must(os.WriteFile(env["PRODUCTION_RELEASE_AUTH_TOKEN_FILE"], []byte(objectgateway.Digest([]byte(root+"approval"))), 0600))
|
||||
cfg, e := appconfig.Load(func(k string) string { return env[k] })
|
||||
must(e)
|
||||
dataToken := objectgateway.Digest([]byte(root + "data"))
|
||||
controlToken := objectgateway.Digest([]byte(root + "control"))
|
||||
env["LIBRARY_OBJECT_GATEWAY_URL"] = "http://" + os.Getenv("KG_R38_GATEWAY_ADDR")
|
||||
env["LIBRARY_OBJECT_GATEWAY_TOKEN"] = dataToken
|
||||
|
||||
// worker 的对象身份是只读的:任何对象写入只能经网关,receipt 因此必然来自真实生产写。
|
||||
readStore, e := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: env["LIBRARY_PRODUCTION_S3_ENDPOINT"],
|
||||
Bucket: fx.bucket, AccessKeyID: env["LIBRARY_PRODUCTION_S3_ACCESS_KEY"], SecretAccessKey: env["LIBRARY_PRODUCTION_S3_SECRET_KEY"],
|
||||
Region: "us-east-1", ForcePathStyle: true})
|
||||
must(e)
|
||||
allowedRead, e := readStore.Open(ctx, "minio", wsLocator("backend-target", "minio", fx.bucket, 10001, "res-target"))
|
||||
must(e)
|
||||
allowedBody, e := io.ReadAll(allowedRead)
|
||||
allowedRead.Close()
|
||||
must(e)
|
||||
if !bytes.Equal(allowedBody, []byte("12345678")) {
|
||||
t.Fatal("readonly identity could not read original bytes")
|
||||
}
|
||||
if denied := readStore.Put(ctx, "minio", wsLocator("backend-target", "minio", fx.bucket, 10001, "r38-denied"), bytes.NewReader([]byte("denied")), 6); denied == nil {
|
||||
t.Fatal("worker readonly identity unexpectedly wrote an object")
|
||||
}
|
||||
save("05-identity.json", map[string]any{"worker_object_identity": "readonly", "gateway_only_writer": true,
|
||||
"bucket": fx.bucket, "index": fx.index, "target": cfg.LibraryTargetIdentity(), "project": project,
|
||||
"readonly_read_bytes": string(allowedBody), "readonly_put_denied": true})
|
||||
|
||||
scope := objectgateway.Digest([]byte("minio\x00" + env["LIBRARY_PRODUCTION_S3_ENDPOINT"] + "\x00" + fx.bucket))
|
||||
pub, priv, e := ed25519.GenerateKey(rand.Reader)
|
||||
must(e)
|
||||
digestFile := func(name string) string {
|
||||
raw, e := os.ReadFile(filepath.Join(root, name))
|
||||
must(e)
|
||||
return objectgateway.Digest(raw)
|
||||
}
|
||||
initialDrain := filepath.Join(root, "initial-drain.json")
|
||||
if _, e := os.Stat(initialDrain); e != nil {
|
||||
t.Fatalf("runner must provide real initial drain evidence: %v", e)
|
||||
}
|
||||
payload, e := json.Marshal(objectgateway.Attestation{Protocol: objectgateway.Protocol, Scope: scope, GatewayID: project,
|
||||
IssuedAt: time.Now().Add(-time.Second), ExpiresAt: time.Now().Add(50 * time.Minute),
|
||||
WriterInventory: digestFile("05-identity.json"), PolicyEvidence: digestFile("05-identity.json"),
|
||||
DenialEvidence: digestFile("05-identity.json"), DrainEvidence: digestFile("initial-drain.json")})
|
||||
must(e)
|
||||
signedRaw, e := json.Marshal(objectgateway.SignedAttestation{Payload: payload, Signature: ed25519.Sign(priv, payload)})
|
||||
must(e)
|
||||
// 审计有效期按真实时钟记录(网关要求不超过 1 小时;这里自己证明没有改系统时间)。
|
||||
var attestationView objectgateway.Attestation
|
||||
must(json.Unmarshal(payload, &attestationView))
|
||||
nowProbe := time.Now()
|
||||
if attestationView.ExpiresAt.Sub(attestationView.IssuedAt) > time.Hour || nowProbe.After(attestationView.ExpiresAt) {
|
||||
t.Fatalf("gateway audit validity invalid: issued=%s expires=%s now=%s",
|
||||
attestationView.IssuedAt, attestationView.ExpiresAt, nowProbe)
|
||||
}
|
||||
save("05a-gateway-audit.json", map[string]any{"issued_at": attestationView.IssuedAt, "expires_at": attestationView.ExpiresAt,
|
||||
"now": nowProbe, "validity": attestationView.ExpiresAt.Sub(attestationView.IssuedAt).String(),
|
||||
"clock_source": "system-clock-unmodified"})
|
||||
must(os.WriteFile(filepath.Join(root, "audit.json"), signedRaw, 0600))
|
||||
|
||||
processes := []map[string]any{}
|
||||
start := func(binary, label string, values map[string]string) *exec.Cmd {
|
||||
t.Helper()
|
||||
cmd := exec.Command(filepath.Join(os.Getenv("KG_R38_BIN"), binary))
|
||||
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")}
|
||||
for k, v := range values {
|
||||
cmd.Env = append(cmd.Env, k+"="+v)
|
||||
}
|
||||
f, e := os.OpenFile(filepath.Join(root, label+".log"), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
|
||||
must(e)
|
||||
cmd.Stdout = f
|
||||
cmd.Stderr = f
|
||||
must(cmd.Start())
|
||||
processes = append(processes, map[string]any{"label": label, "pid": cmd.Process.Pid, "binary": binary})
|
||||
save("06-processes.json", processes)
|
||||
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait(); f.Close() })
|
||||
return cmd
|
||||
}
|
||||
genv := map[string]string{"OBJECT_GATEWAY_LISTEN": os.Getenv("KG_R38_GATEWAY_ADDR"), "OBJECT_GATEWAY_DATABASE_URL": govDSN,
|
||||
"OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL": reader.Config().ConnString(), "OBJECT_GATEWAY_ARCHIVE_ROOT": archiveRoot,
|
||||
"OBJECT_GATEWAY_TARGET_ID": cfg.LibraryTargetIdentity(), "OBJECT_GATEWAY_PROVIDER": "minio",
|
||||
"OBJECT_GATEWAY_ENDPOINT": env["LIBRARY_PRODUCTION_S3_ENDPOINT"], "OBJECT_GATEWAY_BUCKET": fx.bucket,
|
||||
"OBJECT_GATEWAY_REGION": "us-east-1", "OBJECT_GATEWAY_ACCESS_KEY": os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"),
|
||||
"OBJECT_GATEWAY_SECRET_KEY": os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY"), "OBJECT_GATEWAY_DATA_TOKEN": dataToken,
|
||||
"OBJECT_GATEWAY_CONTROL_TOKEN": controlToken, "OBJECT_GATEWAY_ID": project,
|
||||
"OBJECT_GATEWAY_AUDIT_FILE": filepath.Join(root, "audit.json"),
|
||||
"OBJECT_GATEWAY_AUDIT_PUBLIC_KEY": base64.StdEncoding.EncodeToString(pub)}
|
||||
start("object-gateway", "gateway", genv)
|
||||
client, e := objectgateway.NewClient(env["LIBRARY_OBJECT_GATEWAY_URL"], dataToken, scope)
|
||||
must(e)
|
||||
ready := false
|
||||
for i := 0; i < 100; i++ {
|
||||
if client.ReadyForAcquisition(ctx) == nil {
|
||||
ready = true
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if !ready {
|
||||
t.Fatal("object gateway not ready; inspect gateway.log")
|
||||
}
|
||||
|
||||
apiAddr := os.Getenv("KG_R38_API_ADDR")
|
||||
apiEnv := map[string]string{}
|
||||
for k, v := range env {
|
||||
apiEnv[k] = v
|
||||
}
|
||||
apiEnv["APP_ADDR"] = apiAddr
|
||||
apiEnv["GOVERNANCE_SPOOL_ROOT"] = filepath.Join(root, "spool")
|
||||
apiEnv["RELEASE_WORKER_ENABLED"] = "false"
|
||||
must(os.MkdirAll(apiEnv["GOVERNANCE_SPOOL_ROOT"], 0700))
|
||||
start("api", "api", apiEnv)
|
||||
apiURL := "http://" + apiAddr
|
||||
for i := 0; i < 120; i++ {
|
||||
if response, err := http.Get(apiURL + "/health"); err == nil {
|
||||
response.Body.Close()
|
||||
if response.StatusCode == 200 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if i == 119 {
|
||||
t.Fatal("api /health never became 200")
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
// 管理员会话(只改这一次性治理库里的 admin 口令;口令只存在内存里)。
|
||||
password := hex.EncodeToString(func() []byte { b := make([]byte, 18); _, _ = rand.Read(b); return b }())
|
||||
hash, e := bcrypt.GenerateFromPassword([]byte(password), 10)
|
||||
must(e)
|
||||
_, e = gov.Exec(ctx, `UPDATE users SET password_hash=$1, must_change_password=false, failed_attempts=0,
|
||||
locked_until=NULL, state='active' WHERE username='admin'`, string(hash))
|
||||
must(e)
|
||||
jar, e := cookiejar.New(nil)
|
||||
must(e)
|
||||
api := r38API{base: apiURL, client: &http.Client{Jar: jar, Timeout: 30 * time.Second}}
|
||||
code, body := api.do(t, http.MethodPost, "/api/v1/auth/login", map[string]string{"username": "admin", "password": password})
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("admin login => %d %v", code, body)
|
||||
}
|
||||
if user, _ := body["data"].(map[string]any); user["role"] != "admin" {
|
||||
t.Fatalf("admin session role = %v", body)
|
||||
}
|
||||
save("07-session.json", map[string]any{"login_http": code, "role": "admin"})
|
||||
|
||||
// ---------- 3) 播种真实发布作业(worker 执行,测试不写目标) ----------
|
||||
store := postgresstore.NewStore(gov).WithLibraryTarget(cfg.LibraryTargetIdentity())
|
||||
now := time.Now().UTC()
|
||||
digest, e := governance.LibraryRequestDigest("internal", governance.LibraryJobSync, map[string]any{"target_snapshot_id": source})
|
||||
must(e)
|
||||
must(store.CreateLibraryAuthorization(ctx, governance.LibraryReleaseAuthorization{ID: project + "-auth", Scope: "internal",
|
||||
SnapshotID: source, ParamsDigest: digest, ExpiresAt: now.Add(time.Hour), CreatedAt: now}))
|
||||
op, _, e := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{ID: project + "-operation", Scope: "internal",
|
||||
Kind: governance.LibraryJobSync, TargetSnapshotID: source, RequestDigest: digest, OperationKey: project, CreatedAt: now})
|
||||
must(e)
|
||||
job, _, e := store.CreateLibraryDispatch(ctx, governance.LibraryJob{ID: project + "-job", OperationID: op.ID, Scope: "internal",
|
||||
Kind: governance.LibraryJobSync, TargetSnapshotID: source, IdempotencyKey: project, RequestDigest: digest},
|
||||
governance.SyncStages(), false, now)
|
||||
must(e)
|
||||
queueID := job.QueueJobID
|
||||
if queueID == "" {
|
||||
must(gov.QueryRow(ctx, "SELECT queue_job_id FROM library_jobs WHERE id=$1", job.ID).Scan(&queueID))
|
||||
}
|
||||
save("08-job.json", map[string]any{"job": job.ID, "operation": job.OperationID, "queue": queueID,
|
||||
"snapshot": source, "index": fx.index, "bucket": fx.bucket, "target": cfg.LibraryTargetIdentity()})
|
||||
|
||||
workerEnv := map[string]string{}
|
||||
for k, v := range env {
|
||||
workerEnv[k] = v
|
||||
}
|
||||
workerEnv["WORKER_ID"] = project + "-worker"
|
||||
worker := start("worker", "worker", workerEnv)
|
||||
|
||||
waitJob := func(id string, want func(string) bool, seconds int, label string) string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(time.Duration(seconds) * time.Second)
|
||||
var state, message string
|
||||
for time.Now().Before(deadline) {
|
||||
if e := gov.QueryRow(ctx, "SELECT state,COALESCE(last_error,'') FROM jobs WHERE id=$1", id).Scan(&state, &message); e != nil {
|
||||
t.Fatalf("read queue %s: %v", id, e)
|
||||
}
|
||||
if want(state) {
|
||||
return state
|
||||
}
|
||||
if state == "failed" || state == "completed" || state == "cancelled" || state == "dead" {
|
||||
if !want(state) {
|
||||
t.Fatalf("%s terminal state=%s error=%s", label, state, message)
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("%s timeout (state=%s error=%s)", label, state, message)
|
||||
return state
|
||||
}
|
||||
// 真实发布在四个阶段写完后停在 wiki 之后的屏障上;此时才注入一次性"校验读回不一致",
|
||||
// 放行后由生产自己的校验判定失败并进入回滚。
|
||||
hitPath := filepath.Join(root, "barriers", "after-wiki.hit")
|
||||
released := false
|
||||
for i := 0; i < 900; i++ {
|
||||
if _, e := os.Stat(hitPath); e == nil {
|
||||
proxy.armVerifyCorruption()
|
||||
must(os.WriteFile(filepath.Join(root, "barriers", "after-wiki.go"), []byte("release"), 0600))
|
||||
save("08a-barrier.json", map[string]any{"point": "after:wiki", "hit": true, "released": true,
|
||||
"verify_corruption_armed": true, "refuse_clean_dispatch": true})
|
||||
released = true
|
||||
break
|
||||
}
|
||||
var state string
|
||||
must(gov.QueryRow(ctx, "SELECT state FROM jobs WHERE id=$1", queueID).Scan(&state))
|
||||
if state == "failed" || state == "completed" || state == "cancelled" || state == "dead" {
|
||||
t.Fatalf("publish reached %s before the after:wiki barrier", state)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if !released {
|
||||
t.Fatal("publish never reached the after:wiki barrier")
|
||||
}
|
||||
if got := waitJob(queueID, func(s string) bool { return s == "failed" }, 180, "publish rollback must fail at the restore boundary"); got != "failed" {
|
||||
t.Fatalf("publish state = %s", got)
|
||||
}
|
||||
libraryAfterPublish, e := store.LibraryJob(ctx, job.ID)
|
||||
must(e)
|
||||
save("09-publish-failed.json", libraryAfterPublish)
|
||||
if libraryAfterPublish.State != governance.LibraryJobFailed {
|
||||
t.Fatalf("publish library state = %s (%s)", libraryAfterPublish.State, libraryAfterPublish.Error)
|
||||
}
|
||||
failureText := libraryAfterPublish.Error + "\n" + libraryAfterPublish.Recovery
|
||||
if !strings.Contains(failureText, "恢复失败") || !strings.Contains(failureText, "index effect outcome unknown") {
|
||||
t.Fatalf("publish failure does not name the refused rollback write: %q", failureText)
|
||||
}
|
||||
if string(libraryAfterPublish.Stage) != "rollback" {
|
||||
t.Fatalf("publish failed at stage %q, want rollback", libraryAfterPublish.Stage)
|
||||
}
|
||||
if proxy.corruptionCount() != 1 {
|
||||
t.Fatalf("verify corruption applied %d times, want exactly 1", proxy.corruptionCount())
|
||||
}
|
||||
|
||||
// 回滚失败必须体现为"窗口 held + 两个维护记录 needs_restore",而不是只在作业文案里。
|
||||
heldJournal := readR38JSON(t, filepath.Join(root, "maintenance", job.ID+".json"))
|
||||
save("10-journal-held.json", heldJournal)
|
||||
if heldJournal["state"] != "held" {
|
||||
t.Fatalf("freeze journal state = %v, want held", heldJournal["state"])
|
||||
}
|
||||
heldCode, _ := heldJournal["code"].(string)
|
||||
// 回滚路径的恢复失败由窗口自己记为 restore_blocked(RecoverWindow 的同类失败是
|
||||
// restore_failed):两者都表示"目标不可用、内容风险未解除",不是成功收尾。
|
||||
if heldCode != "restore_blocked" {
|
||||
t.Fatalf("freeze journal code = %q, want restore_blocked", heldCode)
|
||||
}
|
||||
if heldJournal["restore_blocked"] != true {
|
||||
t.Fatalf("freeze journal restore_blocked = %v", heldJournal["restore_blocked"])
|
||||
}
|
||||
backupID, _ := heldJournal["backup_snapshot_id"].(string)
|
||||
if backupID == "" {
|
||||
t.Fatal("held journal lost the prebackup identity")
|
||||
}
|
||||
instance, e := store.InstanceMaintenanceOf(ctx, cfg.LibraryTargetInstance)
|
||||
must(e)
|
||||
scopeRow, e := store.LibraryMaintenanceOf(ctx, "internal")
|
||||
must(e)
|
||||
if instance == nil || instance.ReleasedAt != nil || instance.State != "needs_restore" || instance.OwnerJobID != job.ID {
|
||||
t.Fatalf("instance maintenance = %+v", instance)
|
||||
}
|
||||
if scopeRow == nil || scopeRow.ReleasedAt != nil || scopeRow.State != "needs_restore" || scopeRow.OwnerJobID != job.ID {
|
||||
t.Fatalf("scope maintenance = %+v", scopeRow)
|
||||
}
|
||||
save("11-maintenance-held.json", map[string]any{"instance": instance, "scope": scopeRow})
|
||||
|
||||
// 精确边界:restore-finalize-intent 已落盘、marker 原样停在自己世代上、clean 未派发。
|
||||
effects := r38IndexEffects(t, archiveRoot, "chunk-target")
|
||||
save("12-index-effects-held.json", effects)
|
||||
for _, phase := range []string{"apply-intent", "applied", "restore-intent", "restore-claimed", "restore-finalize-intent"} {
|
||||
if _, ok := effects[phase]; !ok {
|
||||
t.Fatalf("held boundary missing %s evidence (have %v)", phase, effectPhases(effects))
|
||||
}
|
||||
}
|
||||
if _, ok := effects["restored"]; ok {
|
||||
t.Fatal("held boundary must not carry a restored receipt")
|
||||
}
|
||||
heldIndex := recoveryIndexReadback(t, fx)
|
||||
save("13-index-held.json", heldIndex)
|
||||
heldDoc, ok := heldIndex["chunk-target"]
|
||||
if !ok {
|
||||
t.Fatal("chunk-target missing at held boundary")
|
||||
}
|
||||
heldSource := heldDoc["_source"].(map[string]any)
|
||||
heldContent, _ := heldSource["content"].(string)
|
||||
if !strings.HasPrefix(heldContent, indexRestoreClaimPrefix) {
|
||||
t.Fatalf("held index must still carry the restore marker, got %q", heldContent)
|
||||
}
|
||||
events, refused := proxy.snapshot()
|
||||
save("14-wire-held.json", map[string]any{"events": events, "refused": refused})
|
||||
if refused != 1 {
|
||||
t.Fatalf("clean dispatch refusals = %d, want exactly 1", refused)
|
||||
}
|
||||
forwarded := 0
|
||||
for _, event := range events {
|
||||
if event["refused"] == true && event["dispatched"] == false {
|
||||
continue
|
||||
}
|
||||
if event["method"] == http.MethodPut && strings.Contains(event["path"].(string), "/_doc/chunk-target") {
|
||||
forwarded++
|
||||
}
|
||||
}
|
||||
if forwarded != 0 {
|
||||
t.Fatalf("refused clean write was forwarded %d times", forwarded)
|
||||
}
|
||||
|
||||
// HTTP 入口:窗口 held ⇒ 读写入口 409;恢复入口仍是唯一出口。
|
||||
code, maintenance := api.do(t, http.MethodGet, "/api/v1/library/maintenance?scope=internal", nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("GET maintenance => %d %v", code, maintenance)
|
||||
}
|
||||
save("15-maintenance-http-held.json", map[string]any{"http": code, "body": maintenance})
|
||||
if maintenance["held"] != true {
|
||||
t.Fatalf("maintenance HTTP held = %v", maintenance["held"])
|
||||
}
|
||||
instanceView, _ := maintenance["instance_maintenance"].(map[string]any)
|
||||
if instanceView == nil || instanceView["state"] != "needs_restore" || instanceView["owner_job_id"] != job.ID {
|
||||
t.Fatalf("maintenance HTTP instance view = %v", maintenance["instance_maintenance"])
|
||||
}
|
||||
// held 期间入口必须真的关着:连不上,而不是"还有人返回错误页"。
|
||||
closedClient := &http.Client{Timeout: 3 * time.Second}
|
||||
entryOpen := false
|
||||
entryDetail := ""
|
||||
if response, err := closedClient.Get(os.Getenv("KG_R38_INGRESS")); err == nil {
|
||||
entryOpen = response.StatusCode == http.StatusOK
|
||||
entryDetail = fmt.Sprintf("HTTP %d", response.StatusCode)
|
||||
response.Body.Close()
|
||||
} else {
|
||||
entryDetail = err.Error()
|
||||
}
|
||||
save("15a-ingress-held.json", map[string]any{"ingress": os.Getenv("KG_R38_INGRESS"), "open": entryOpen, "detail": entryDetail})
|
||||
if entryOpen {
|
||||
t.Fatalf("target ingress still serves 200 while the window is held: %s", entryDetail)
|
||||
}
|
||||
code, blocked := api.do(t, http.MethodPost, "/api/v1/library/restores", map[string]any{"scope": "internal", "snapshot_id": backupID})
|
||||
if code != http.StatusConflict || blocked["code"] != "library_target_in_maintenance" {
|
||||
t.Fatalf("entry while held => %d %v, want 409 library_target_in_maintenance", code, blocked)
|
||||
}
|
||||
save("16-entry-409.json", map[string]any{"http": code, "body": blocked})
|
||||
code, mismatch := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
|
||||
map[string]any{"scope": "internal", "window_job_id": "some-other-window"})
|
||||
if code != http.StatusConflict || mismatch["code"] != "library_window_mismatch" {
|
||||
t.Fatalf("recover with another window => %d %v", code, mismatch)
|
||||
}
|
||||
save("17-window-binding-409.json", map[string]any{"http": code, "body": mismatch})
|
||||
|
||||
// ---------- 4) 撤故障 → 正常 POST /library/maintenance/recover ----------
|
||||
proxy.setRefuseClean(false)
|
||||
code, accepted := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
|
||||
map[string]any{"scope": "internal", "window_job_id": job.ID})
|
||||
if code != http.StatusAccepted {
|
||||
t.Fatalf("POST recover => %d %v", code, accepted)
|
||||
}
|
||||
recoveryJob, _ := accepted["job"].(map[string]any)
|
||||
recoveryQueueID, _ := recoveryJob["id"].(string)
|
||||
save("18-recover-accepted.json", accepted)
|
||||
if accepted["window_job_id"] != job.ID || accepted["scope"] != "internal" {
|
||||
t.Fatalf("recover response window binding = %v", accepted)
|
||||
}
|
||||
if recoveryQueueID != governance.LibraryRecoveryQueueID(job.ID) {
|
||||
t.Fatalf("recovery queue id = %q", recoveryQueueID)
|
||||
}
|
||||
if got := waitJob(recoveryQueueID, func(s string) bool { return s == "completed" }, 300, "recovery must complete"); got != "completed" {
|
||||
t.Fatalf("recovery state = %s", got)
|
||||
}
|
||||
|
||||
recoveredJournal := readR38JSON(t, filepath.Join(root, "maintenance", job.ID+".json"))
|
||||
save("19-journal-recovered.json", recoveredJournal)
|
||||
if recoveredJournal["state"] != "recovered" {
|
||||
t.Fatalf("journal after recovery = %v", recoveredJournal["state"])
|
||||
}
|
||||
if recoveredJournal["recovered_backup"] != backupID || recoveredJournal["backup_snapshot_id"] != backupID {
|
||||
t.Fatalf("recovery bound another backup: %v", recoveredJournal)
|
||||
}
|
||||
if digest, _ := recoveredJournal["recovery_digest"].(string); digest == "" {
|
||||
t.Fatal("recovery finished without a verification digest")
|
||||
}
|
||||
instanceAfter, e := store.InstanceMaintenanceOf(ctx, cfg.LibraryTargetInstance)
|
||||
must(e)
|
||||
scopeAfter, e := store.LibraryMaintenanceOf(ctx, "internal")
|
||||
must(e)
|
||||
if instanceAfter == nil || instanceAfter.ReleasedAt == nil {
|
||||
t.Fatalf("instance maintenance not released: %+v", instanceAfter)
|
||||
}
|
||||
if scopeAfter == nil || scopeAfter.ReleasedAt == nil {
|
||||
t.Fatalf("scope maintenance not released: %+v", scopeAfter)
|
||||
}
|
||||
save("20-maintenance-released.json", map[string]any{"instance": instanceAfter, "scope": scopeAfter})
|
||||
code, openMaintenance := api.do(t, http.MethodGet, "/api/v1/library/maintenance?scope=internal", nil)
|
||||
if code != http.StatusOK || openMaintenance["held"] != false {
|
||||
t.Fatalf("maintenance after recovery => %d %v", code, openMaintenance)
|
||||
}
|
||||
// 入口真的回来了:实际 HTTP 200,而不是"容器起来了"。
|
||||
probe := false
|
||||
for i := 0; i < 100; i++ {
|
||||
response, err := http.Get(os.Getenv("KG_R38_INGRESS"))
|
||||
if err == nil {
|
||||
response.Body.Close()
|
||||
if response.StatusCode == http.StatusOK {
|
||||
probe = true
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
if !probe {
|
||||
t.Fatal("target ingress did not return HTTP 200 after recovery")
|
||||
}
|
||||
save("21-ingress.json", map[string]any{"ingress": os.Getenv("KG_R38_INGRESS"), "http": 200, "open": true,
|
||||
"maintenance_http": openMaintenance})
|
||||
|
||||
// ---------- 5) 三个真实存储的全量恢复 ----------
|
||||
afterPG := universeDump(t, fx.pool)
|
||||
save("22-pg-after.json", afterPG)
|
||||
if diff := diffUniverse(beforePG, afterPG); diff != "" {
|
||||
t.Fatalf("PG not restored row-by-row: %s", diff)
|
||||
}
|
||||
afterObjects := objectInventoryR38(t, ctx, fx)
|
||||
save("23-objects-after.json", afterObjects)
|
||||
if !reflect.DeepEqual(objectVersionIDs(beforeObjects), objectVersionIDs(afterObjects)) {
|
||||
t.Fatalf("object versions differ from before-image:\n before=%v\n after=%v",
|
||||
objectVersionIDs(beforeObjects), objectVersionIDs(afterObjects))
|
||||
}
|
||||
afterIndex := recoveryIndexReadback(t, fx)
|
||||
save("24-index-after.json", afterIndex)
|
||||
if len(afterIndex) != len(beforeIndex) {
|
||||
t.Fatalf("index document set changed: %d -> %d", len(beforeIndex), len(afterIndex))
|
||||
}
|
||||
for id, old := range beforeIndex {
|
||||
got, ok := afterIndex[id]
|
||||
if !ok || !reflect.DeepEqual(old["_source"], got["_source"]) {
|
||||
t.Fatalf("index document %s not restored to its before-image", id)
|
||||
}
|
||||
if id == "chunk-target" && reflect.DeepEqual(old["_seq_no"], got["_seq_no"]) {
|
||||
t.Fatal("restoration reused the applied generation instead of writing a new one")
|
||||
}
|
||||
}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
|
||||
t.Fatalf("independent verification after recovery: %v", err)
|
||||
}
|
||||
recoveredEffects := r38IndexEffects(t, archiveRoot, "chunk-target")
|
||||
save("25-index-effects-recovered.json", recoveredEffects)
|
||||
if _, ok := recoveredEffects["restored"]; !ok {
|
||||
t.Fatalf("recovery did not persist a restored receipt (have %v)", effectPhases(recoveredEffects))
|
||||
}
|
||||
|
||||
// 恢复事实与 receipt 都由真实写入产生,没有人工造 receipt。
|
||||
var recoverOps, recoveryVersions int
|
||||
must(gov.QueryRow(ctx, "SELECT count(*) FROM library_operations WHERE kind='recover'").Scan(&recoverOps))
|
||||
must(gov.QueryRow(ctx, "SELECT count(*) FROM library_versions WHERE kind='recovery_verified' AND source_job_id=$1", job.ID).Scan(&recoveryVersions))
|
||||
var prebackups int
|
||||
must(gov.QueryRow(ctx, "SELECT count(*) FROM library_versions WHERE kind='production_prebackup' AND source_job_id=$1 AND snapshot_id=$2",
|
||||
job.ID, backupID).Scan(&prebackups))
|
||||
if recoverOps != 1 || recoveryVersions != 1 || prebackups != 1 {
|
||||
t.Fatalf("durable recovery identities ops=%d events=%d prebackups=%d", recoverOps, recoveryVersions, prebackups)
|
||||
}
|
||||
receipts, deliveries := r38Receipts(t, ctx, gov, fx)
|
||||
save("26-receipts.json", map[string]any{"receipts": receipts, "deliveries": deliveries})
|
||||
if len(receipts) == 0 || len(deliveries) == 0 {
|
||||
t.Fatal("no gateway receipt/delivery evidence for real object writes")
|
||||
}
|
||||
// receipt 必须来自真实网关写入:每条都有对应的 delivery(同计划摘要、同代次),
|
||||
// 键落在本目标桶的命名空间里;被回滚删掉的那个版本必须有同一网关的 removed 回执。
|
||||
committed := 0
|
||||
for _, receipt := range receipts {
|
||||
if receipt.Request.Key == "" || receipt.Version.ID == "" || receipt.Request.Grant.Plan == "" {
|
||||
t.Fatalf("incomplete receipt %+v", receipt.Request)
|
||||
}
|
||||
matched := false
|
||||
for _, delivery := range deliveries {
|
||||
if delivery.PlanDigest == receipt.Request.Grant.Plan && delivery.JobID != "" && delivery.AttemptEpoch == receipt.Request.Grant.Epoch {
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
t.Fatalf("receipt without a durable delivery (plan=%s epoch=%d)", receipt.Request.Grant.Plan, receipt.Request.Grant.Epoch)
|
||||
}
|
||||
// 键必须落在本目标桶的对象命名空间里(10001/ 是目标租户前缀)。
|
||||
if !strings.HasPrefix(receipt.Request.Key, "10001/") {
|
||||
t.Fatalf("receipt key %q is outside the isolated target namespace", receipt.Request.Key)
|
||||
}
|
||||
if receipt.State != "committed" {
|
||||
continue
|
||||
}
|
||||
committed++
|
||||
present := false
|
||||
for _, version := range afterObjects.Versions {
|
||||
if aws.ToString(version.Key) == receipt.Request.Key && aws.ToString(version.VersionId) == receipt.Version.ID {
|
||||
present = true
|
||||
}
|
||||
}
|
||||
if present {
|
||||
continue
|
||||
}
|
||||
removed := false
|
||||
for _, other := range receipts {
|
||||
if other.State == "removed" && other.Request.Key == receipt.Request.Key && other.Version.ID == receipt.Version.ID {
|
||||
removed = true
|
||||
}
|
||||
}
|
||||
if !removed {
|
||||
t.Fatalf("committed receipt %s@%s vanished without a gateway removal receipt",
|
||||
receipt.Request.Key, receipt.Version.ID)
|
||||
}
|
||||
}
|
||||
if committed == 0 {
|
||||
t.Fatal("no committed gateway receipt: object writes cannot be tied to receipt evidence")
|
||||
}
|
||||
// Recovery 请求重放:幂等,不得再派发任何写入。
|
||||
code, replay := api.do(t, http.MethodPost, "/api/v1/library/maintenance/recover",
|
||||
map[string]any{"scope": "internal", "window_job_id": job.ID})
|
||||
if code != http.StatusConflict || replay["code"] != "library_no_held_window" {
|
||||
t.Fatalf("recover replay on a released window => %d %v", code, replay)
|
||||
}
|
||||
save("27-recover-replay.json", map[string]any{"http": code, "body": replay})
|
||||
if !reflect.DeepEqual(afterPG, universeDump(t, fx.pool)) {
|
||||
t.Fatal("replay mutated PG")
|
||||
}
|
||||
if !reflect.DeepEqual(afterIndex, recoveryIndexReadback(t, fx)) {
|
||||
t.Fatal("replay rewrote index generations")
|
||||
}
|
||||
|
||||
must(worker.Process.Signal(os.Interrupt))
|
||||
_ = worker.Wait()
|
||||
save("28-result.json", map[string]any{"window_job": job.ID, "publish_queue": queueID,
|
||||
"recovery_queue": recoveryQueueID, "backup": backupID, "scope": "internal", "instance": cfg.LibraryTargetInstance,
|
||||
"held": map[string]any{"journal": "held/restore_blocked", "maintenance": "needs_restore", "entry_http": 409},
|
||||
"recovered": map[string]any{"journal": "recovered", "maintenance_released": true, "entry_http": 200,
|
||||
"pg_rows_restored": true, "object_versions_restored": true, "es_source_restored": true,
|
||||
"receipts": len(receipts), "deliveries": len(deliveries), "recover_operations": recoverOps},
|
||||
"refusals": refused})
|
||||
t.Logf("r38 closed loop: window=%s backup=%s recovery=%s receipts=%d deliveries=%d",
|
||||
job.ID, backupID, recoveryQueueID, len(receipts), len(deliveries))
|
||||
}
|
||||
|
||||
// ---------- 观察辅助 ----------
|
||||
|
||||
func objectInventoryR38(t *testing.T, ctx context.Context, fx *writeSetFixture) *s3.ListObjectVersionsOutput {
|
||||
t.Helper()
|
||||
out, e := fx.objects.s3.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(fx.bucket)})
|
||||
if e != nil {
|
||||
t.Fatalf("object inventory: %v", e)
|
||||
}
|
||||
if aws.ToBool(out.IsTruncated) || len(out.DeleteMarkers) != 0 {
|
||||
t.Fatal("incomplete object inventory or unexpected delete markers")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func objectVersionIDs(inventory *s3.ListObjectVersionsOutput) []string {
|
||||
ids := make([]string, 0, len(inventory.Versions))
|
||||
for _, version := range inventory.Versions {
|
||||
ids = append(ids, aws.ToString(version.Key)+"@"+aws.ToString(version.VersionId)+"#"+aws.ToString(version.ETag))
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
func readR38JSON(t *testing.T, path string) map[string]any {
|
||||
t.Helper()
|
||||
raw, e := os.ReadFile(path)
|
||||
if e != nil {
|
||||
t.Fatalf("read %s: %v", path, e)
|
||||
}
|
||||
out := map[string]any{}
|
||||
if e = json.Unmarshal(raw, &out); e != nil {
|
||||
t.Fatalf("parse %s: %v", path, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// r38IndexEffects 汇总某个文档在冻结计划下已落盘的阶段证据(phase -> 证据内容)。
|
||||
func r38IndexEffects(t *testing.T, archiveRoot, id string) map[string]map[string]any {
|
||||
t.Helper()
|
||||
out := map[string]map[string]any{}
|
||||
dir := filepath.Join(archiveRoot, "effects")
|
||||
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info == nil || info.IsDir() || !strings.HasSuffix(path, ".json") {
|
||||
return nil
|
||||
}
|
||||
raw, e := os.ReadFile(path)
|
||||
if e != nil {
|
||||
return nil
|
||||
}
|
||||
var effect struct {
|
||||
Change map[string]any `json:"change"`
|
||||
Receipt map[string]any `json:"receipt"`
|
||||
}
|
||||
if json.Unmarshal(raw, &effect) != nil || effect.Change == nil {
|
||||
return nil
|
||||
}
|
||||
if got, _ := effect.Change["id"].(string); got != id {
|
||||
return nil
|
||||
}
|
||||
phase := strings.TrimSuffix(filepath.Base(path), ".json")
|
||||
out[phase] = map[string]any{"change": effect.Change, "receipt": effect.Receipt}
|
||||
return nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func effectPhases(effects map[string]map[string]any) []string {
|
||||
phases := make([]string, 0, len(effects))
|
||||
for phase := range effects {
|
||||
phases = append(phases, phase)
|
||||
}
|
||||
sort.Strings(phases)
|
||||
return phases
|
||||
}
|
||||
|
||||
func r38Receipts(t *testing.T, ctx context.Context, gov *pgxpool.Pool, fx *writeSetFixture) ([]objectgateway.Receipt, []objectgateway.Claim) {
|
||||
t.Helper()
|
||||
rows, e := gov.Query(ctx, "SELECT receipt FROM kg_object_gateway_receipts ORDER BY 1")
|
||||
if e != nil {
|
||||
t.Fatalf("receipts: %v", e)
|
||||
}
|
||||
receipts := []objectgateway.Receipt{}
|
||||
for rows.Next() {
|
||||
var raw []byte
|
||||
if e := rows.Scan(&raw); e != nil {
|
||||
rows.Close()
|
||||
t.Fatalf("scan receipt: %v", e)
|
||||
}
|
||||
var receipt objectgateway.Receipt
|
||||
if e := json.Unmarshal(raw, &receipt); e != nil {
|
||||
rows.Close()
|
||||
t.Fatalf("parse receipt: %v", e)
|
||||
}
|
||||
receipts = append(receipts, receipt)
|
||||
}
|
||||
rows.Close()
|
||||
deliveryRows, e := gov.Query(ctx, "SELECT claim FROM kg_object_gateway_deliveries ORDER BY attempt_epoch")
|
||||
if e != nil {
|
||||
t.Fatalf("deliveries: %v", e)
|
||||
}
|
||||
deliveries := []objectgateway.Claim{}
|
||||
for deliveryRows.Next() {
|
||||
var raw []byte
|
||||
if e := deliveryRows.Scan(&raw); e != nil {
|
||||
deliveryRows.Close()
|
||||
t.Fatalf("scan delivery: %v", e)
|
||||
}
|
||||
var claim objectgateway.Claim
|
||||
if e := json.Unmarshal(raw, &claim); e != nil {
|
||||
deliveryRows.Close()
|
||||
t.Fatalf("parse delivery: %v", e)
|
||||
}
|
||||
deliveries = append(deliveries, claim)
|
||||
}
|
||||
deliveryRows.Close()
|
||||
return receipts, deliveries
|
||||
}
|
||||
Reference in New Issue
Block a user