恢复/回滚的索引步骤如果在"持久意图落盘之后、派发之前"确定失败,重试会被判成 unknown restoration lacks exact operation marker/content/CAS 而永久 held:代码 只看标记在不在,没看"这份意图到底发出去没有"。新增 restorationProvablyUnsent: 观察到的内容仍等于自己确认过的 applied 结果(删除即缺失)、且世代与那份回执完全 一致时,成功写入必然产生新世代这一事实就证明恢复写入一次都没到达索引,按原 before-image 重做不会覆盖任何别人的字节。内容对但世代对不上、或应用期停在标记态 却对不上标记体/世代,仍然 held(B19 不放宽)。 新增一次性恢复写入派发边界 restoreMarkerFault(生产恒为 nil),用来把"边界上确定 没写进去"与"写入结果不确定"分开演练。 验:新增 TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated 在真实 ES/PostgreSQL/MinIO 上跑:故障前 applied 回执已落盘、失败时派发次数 0、状态为 rollback_failed;撤故障后同一 job/预备份恢复完成,PG 逐行字节、索引 ID+字节+新世代、 对象键集合全部回到 before-image,重放不再派发。RED 证据是重试报 held,GREEN 后 TestContract(Index|KnownIndex).*Isolated 11 项与两条 rollback 用例全绿,make verify success。
This commit is contained in:
@@ -1,5 +1,18 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照(索引恢复在"确定没写进去"时可按证据重做)
|
||||
|
||||
### Fixed
|
||||
|
||||
- 恢复/回滚的索引步骤若在**持久意图之后、派发之前**确定失败(控制流拒绝),重试不再被永久判成
|
||||
`unknown restoration lacks exact operation marker/content/CAS` 而 held:只要观察到的内容仍等于自己
|
||||
确认过的 applied 结果(删除即缺失)、且世代与那份回执完全一致,就说明这份恢复意图一次都没到达
|
||||
索引,按原 before-image 重做不会覆盖任何别人的字节。
|
||||
- 判据仍然只有世代,不认"内容看起来一样":内容对但世代对不上、应用期停在标记态却对不上标记体或
|
||||
世代,一律保持 held(B19 不放宽)。
|
||||
- 新增一次性恢复写入派发边界(`restoreMarkerFault`,生产恒为 nil),用来把"边界上确定没写进去"与
|
||||
"写入结果不确定"分开演练;真实 ES / PostgreSQL / 对象存储仍是唯一权威。
|
||||
|
||||
## 开发快照(归档改成不可覆盖副本,current 不再先被移走)
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -87,6 +87,16 @@ func sameIndexGeneration(g indexWriteGuard, seq, term *int64) bool {
|
||||
return g.Found && g.SeqNo != nil && g.PrimaryTerm != nil && seq != nil && term != nil && *g.SeqNo == *seq && *g.PrimaryTerm == *term
|
||||
}
|
||||
|
||||
// refuseRestoreWrite 是恢复写入的派发边界(生产恒为 nil):验收用例在这里制造
|
||||
// "边界上确定失败"——方法在派发前返回错误,真实 ES 一个字节都收不到,这与断线、
|
||||
// 响应丢失(写入结果未知,按 B19 必须 held)是两件不同的事。
|
||||
func (t *ProductionLibraryTarget) refuseRestoreWrite(c libraryrelease.IndexWrite) error {
|
||||
if t.restoreMarkerFault == nil {
|
||||
return nil
|
||||
}
|
||||
return t.restoreMarkerFault(c)
|
||||
}
|
||||
|
||||
// Each network mutation has an immutable intent before send and a per-ID
|
||||
// server generation receipt. A lost response is UNKNOWN, never "did not write".
|
||||
// No retry is issued for an unresolved intent; downstream ownership fencing is
|
||||
@@ -271,6 +281,9 @@ func (t *ProductionLibraryTarget) restorePlannedIndex(ctx context.Context) error
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := t.refuseRestoreWrite(c); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm, len(c.After) == 0)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// restoreMarkerWire 只旁路计数:真正派发到真实 ES 的"恢复标记写入"次数。
|
||||
//
|
||||
// 为什么要有它:本用例要证明第一次失败发生在**派发之前**(控制流在边界上拒绝),
|
||||
// 不是断线、也不是响应丢失。只靠错误文本分不出这两者,次数能分:边界拒绝 = 0 次派发,
|
||||
// 断线 = 1 次派发且结果未知。它不改写请求与响应,ES 仍是唯一权威。
|
||||
type restoreMarkerWire struct {
|
||||
inner http.RoundTripper
|
||||
dispatch int
|
||||
}
|
||||
|
||||
func (w *restoreMarkerWire) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
raw, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(raw))
|
||||
if r.Method == http.MethodPut && bytes.Contains(raw, []byte(indexRestoreClaimPrefix)) {
|
||||
w.dispatch++
|
||||
}
|
||||
}
|
||||
return w.inner.RoundTrip(r)
|
||||
}
|
||||
|
||||
// objectKeySet 是对象存储的精确集合证据:键 -> 字节长度/ETag。
|
||||
// 比"某个对象在不在"强:多一个残留键、少一个原有键、或原有对象被改写都会红。
|
||||
func objectKeySet(t *testing.T, fx *writeSetFixture) map[string]string {
|
||||
t.Helper()
|
||||
out := map[string]string{}
|
||||
var token *string
|
||||
for {
|
||||
list, err := fx.objects.s3.client.ListObjectsV2(fx.ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(fx.bucket), ContinuationToken: token})
|
||||
if err != nil {
|
||||
t.Fatalf("list objects: %v", err)
|
||||
}
|
||||
for _, object := range list.Contents {
|
||||
out[aws.ToString(object.Key)] = aws.ToString(object.ETag) + "/" + strconv.FormatInt(aws.ToInt64(object.Size), 10)
|
||||
}
|
||||
if !aws.ToBool(list.IsTruncated) {
|
||||
return out
|
||||
}
|
||||
token = list.NextContinuationToken
|
||||
}
|
||||
}
|
||||
|
||||
func indexChangeOf(t *testing.T, plan *libraryrelease.WriteSet, id string) libraryrelease.IndexWrite {
|
||||
t.Helper()
|
||||
if plan == nil {
|
||||
t.Fatal("write set missing: recovery would have no bound plan")
|
||||
}
|
||||
for _, change := range plan.Index {
|
||||
if change.ID == id {
|
||||
return change
|
||||
}
|
||||
}
|
||||
t.Fatalf("index plan lacks %s", id)
|
||||
return libraryrelease.IndexWrite{}
|
||||
}
|
||||
|
||||
// indexEffectFile 与生产代码的落盘路径同构(sha256(index\x00id)),用于直接读证据文件。
|
||||
func indexEffectFile(fx *writeSetFixture, plan *libraryrelease.WriteSet, change libraryrelease.IndexWrite, phase string) string {
|
||||
sum := sha256.Sum256([]byte(change.Index + "\x00" + change.ID))
|
||||
return filepath.Join(fx.archiveRoot, "effects", plan.Digest, "index", hex.EncodeToString(sum[:]), phase+".json")
|
||||
}
|
||||
|
||||
func requireIndexEffect(t *testing.T, fx *writeSetFixture, plan *libraryrelease.WriteSet, change libraryrelease.IndexWrite, phase string, want bool) string {
|
||||
t.Helper()
|
||||
path := indexEffectFile(fx, plan, change, phase)
|
||||
info, err := os.Stat(path)
|
||||
if want && err != nil {
|
||||
t.Fatalf("index effect %s/%s not durable: %v", change.ID, phase, err)
|
||||
}
|
||||
if !want && err == nil {
|
||||
t.Fatalf("index effect %s/%s exists unexpectedly: %s", change.ID, phase, info.Name())
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// 目标:已知索引效果(真实回执已落盘)在 rollback 的恢复步骤上"确定失败"之后,
|
||||
// 撤掉故障必须能把同一 job/预备份的原 before-image 恢复完成,且 PG / 对象 / 索引
|
||||
// 三个真实存储的集合精确回到预备份时刻。
|
||||
//
|
||||
// 故障边界(target.restoreMarkerFault):在恢复标记的持久意图落盘之后、网络派发之前
|
||||
// 返回错误。这是**控制流在边界上拒绝**,不是断线,也不是响应丢失:真实 ES 一个字节都
|
||||
// 没收到(用例用派发次数 0 证明)。因此重试必须凭"内容仍等于 applied 结果 + 世代仍等于
|
||||
// 自己那份回执"证明第一次没写进去,然后安全重做;而不是把它当成 B19 的不可归属
|
||||
// unknown 永久 held(那种情况仍必须 held,本用例不放宽)。
|
||||
func TestContractKnownIndexRecoveryAfterRestoreBoundaryFaultIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
beforePG := universeDump(t, fx.pool)
|
||||
beforeIndex := recoveryIndexReadback(t, fx)
|
||||
beforeObjects := objectKeySet(t, fx)
|
||||
snapshot := sealSourceSnapshot(t, fx, nil)
|
||||
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
client := *target.search.client
|
||||
inner := client.Transport
|
||||
if inner == nil {
|
||||
inner = http.DefaultTransport
|
||||
}
|
||||
wire := &restoreMarkerWire{inner: inner}
|
||||
client.Transport = wire
|
||||
target.search.client = &client
|
||||
|
||||
var faulted []string
|
||||
target.restoreMarkerFault = func(c libraryrelease.IndexWrite) error {
|
||||
if c.ID != "chunk-target" {
|
||||
return nil
|
||||
}
|
||||
faulted = append(faulted, c.ID)
|
||||
return errors.New("injected restore boundary fault: dispatch refused before any index write")
|
||||
}
|
||||
|
||||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshot), contractFailWikiTarget{target}, backups, nil)
|
||||
if outcome.State != "rollback_failed" {
|
||||
t.Fatalf("state = %q (rollback=%q error=%q), want rollback_failed: a failed index restoration must be held, never admitted",
|
||||
outcome.State, outcome.Rollback, outcome.Error)
|
||||
}
|
||||
if outcome.FailedAt != libraryrelease.StageWiki {
|
||||
t.Fatalf("failed_at = %q, want wiki (the injected apply failure)", outcome.FailedAt)
|
||||
}
|
||||
if !strings.Contains(outcome.Rollback, "injected restore boundary fault") {
|
||||
t.Fatalf("rollback failure does not name the injected index boundary fault: %q", outcome.Rollback)
|
||||
}
|
||||
if len(faulted) != 1 || faulted[0] != "chunk-target" {
|
||||
t.Fatalf("fault fired %v, want exactly one chunk-target restore attempt", faulted)
|
||||
}
|
||||
if wire.dispatch != 0 {
|
||||
t.Fatalf("boundary fault dispatched %d restore marker writes; a control-flow refusal must send nothing", wire.dispatch)
|
||||
}
|
||||
|
||||
// 应用期的真实逐 ID 回执必须已在故障之前落盘:恢复读的是证据,不是目标现状。
|
||||
plan := outcome.Backup.WriteSet
|
||||
if plan == nil || plan.JobID != requestFor(snapshot).JobID {
|
||||
t.Fatalf("backup is not bound to the original job: %+v", plan)
|
||||
}
|
||||
applied, err := filepath.Glob(filepath.Join(fx.archiveRoot, "effects", plan.Digest, "index", "*", "applied.json"))
|
||||
if err != nil || len(applied) != 2 {
|
||||
t.Fatalf("applied index receipts = %d (%v), want 2 durable per-ID receipts", len(applied), err)
|
||||
}
|
||||
change := indexChangeOf(t, plan, "chunk-target")
|
||||
if len(change.Before) == 0 {
|
||||
t.Fatal("fixture lost the chunk-target before-image")
|
||||
}
|
||||
requireIndexEffect(t, fx, plan, change, "apply-intent", true)
|
||||
requireIndexEffect(t, fx, plan, change, "applied", true)
|
||||
// 恢复意图已落盘,但恢复标记从未派发:这正是"可归属、且证明没写过"的状态。
|
||||
requireIndexEffect(t, fx, plan, change, "restore-mark-intent", true)
|
||||
requireIndexEffect(t, fx, plan, change, "restore-claimed", false)
|
||||
requireIndexEffect(t, fx, plan, change, "restored", false)
|
||||
|
||||
// held 期间业务存储不许被"顺手恢复一半再报成功":索引仍是应用后的样子。
|
||||
heldIndex := recoveryIndexReadback(t, fx)
|
||||
if _, ok := heldIndex["chunk-src"]; ok {
|
||||
t.Fatal("index still carries the applied chunk-src after a failed index restoration")
|
||||
}
|
||||
if _, ok := heldIndex["chunk-target"]; ok {
|
||||
t.Fatal("chunk-target came back without the restoration marker ever being dispatched")
|
||||
}
|
||||
|
||||
// 撤掉故障:同一 job / 同一预备份 / 同一 before-image 重新执行恢复。
|
||||
target.restoreMarkerFault = nil
|
||||
if err := target.Restore(fx.ctx, outcome.Backup); err != nil {
|
||||
t.Fatalf("recovery after clearing the fault must settle the durable intent, got: %v", err)
|
||||
}
|
||||
if wire.dispatch != 1 {
|
||||
t.Fatalf("recovery dispatched %d restore marker writes, want exactly 1", wire.dispatch)
|
||||
}
|
||||
|
||||
// 真实 PG:整实例逐行字节回到预备份时刻。
|
||||
if diff := diffUniverse(beforePG, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("post-recovery PG differs from the before-image: %s", diff)
|
||||
}
|
||||
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
|
||||
t.Fatalf("post-recovery verification: %v", err)
|
||||
}
|
||||
|
||||
// 真实索引:ID 集合精确一致、字节一致,恢复产生新世代(不是旧世代冒充)。
|
||||
afterIndex := recoveryIndexReadback(t, fx)
|
||||
if len(afterIndex) != len(beforeIndex) {
|
||||
t.Fatalf("index ID set changed: %d -> %d", len(beforeIndex), len(afterIndex))
|
||||
}
|
||||
for id, old := range beforeIndex {
|
||||
got, ok := afterIndex[id]
|
||||
if !ok || !reflect.DeepEqual(old["_source"], got["_source"]) {
|
||||
t.Fatalf("index document %s not restored to its before-image", id)
|
||||
}
|
||||
if id == "chunk-target" && reflect.DeepEqual(old["_seq_no"], got["_seq_no"]) {
|
||||
t.Fatal("restoration reused the applied generation instead of writing a new one")
|
||||
}
|
||||
}
|
||||
|
||||
// 真实对象存储:精确键集合 + 每个对象字节身份回到预备份时刻。
|
||||
if afterObjects := objectKeySet(t, fx); !reflect.DeepEqual(beforeObjects, afterObjects) {
|
||||
t.Fatalf("object set differs from the before-image:\n before=%v\n after=%v", beforeObjects, afterObjects)
|
||||
}
|
||||
|
||||
// 恢复回执与观察到的世代逐位一致:证据不是"跑完了",而是可复核的服务器回执。
|
||||
restored, err := target.loadIndexEffect(change, "restored")
|
||||
if err != nil || restored == nil || restored.Receipt == nil {
|
||||
t.Fatalf("restored receipt missing: %v", err)
|
||||
}
|
||||
requireIndexEffect(t, fx, plan, change, "restore-claimed", true)
|
||||
observed, err := target.observePlannedIndex(fx.ctx, change)
|
||||
if err != nil || !sameIndexGeneration(observed, restored.Receipt.SeqNo, restored.Receipt.PrimaryTerm) {
|
||||
t.Fatalf("restored receipt does not match the live generation: %v", err)
|
||||
}
|
||||
if !indexBytesEqual(observed, change.Before) {
|
||||
t.Fatal("restored generation does not carry the frozen before-image bytes")
|
||||
}
|
||||
|
||||
// 新目标重放同一份预备份:不得重复派发恢复写入,也不得改动任何集合。
|
||||
next := fx.target(t)
|
||||
if err := next.Restore(fx.ctx, outcome.Backup); err != nil {
|
||||
t.Fatalf("replayed restoration: %v", err)
|
||||
}
|
||||
if replay := recoveryIndexReadback(t, fx); !reflect.DeepEqual(afterIndex, replay) {
|
||||
t.Fatal("replayed restoration rewrote index generations")
|
||||
}
|
||||
if replayObjects := objectKeySet(t, fx); !reflect.DeepEqual(beforeObjects, replayObjects) {
|
||||
t.Fatal("replayed restoration changed the object set")
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,33 @@ import (
|
||||
|
||||
const indexRestoreClaimPrefix = "kg-release-transient-restore-claim:"
|
||||
|
||||
// restorationProvablyUnsent 判定"这份恢复意图一次都没到达索引"。
|
||||
//
|
||||
// 判据只有世代:观察到的内容必须仍是我们确认过的 applied 结果(删除即缺失,
|
||||
// 否则字节与 applied 回执一致),且世代与那份回执完全相同。成功写入必然产生新
|
||||
// 世代,所以世代没变就证明恢复写入没有落地——按证据重做不会覆盖别人的字节。
|
||||
// 反过来,任何"内容看起来对、世代对不上"的状态都只能 held(B19),不认领。
|
||||
func (t *ProductionLibraryTarget) restorationProvablyUnsent(observed indexWriteGuard, c libraryrelease.IndexWrite, applied indexReceipt) (bool, error) {
|
||||
if len(c.After) == 0 {
|
||||
return !observed.Found, nil
|
||||
}
|
||||
if !sameIndexGeneration(observed, applied.SeqNo, applied.PrimaryTerm) {
|
||||
return false, nil
|
||||
}
|
||||
if indexBytesEqual(observed, c.After) {
|
||||
return true, nil
|
||||
}
|
||||
if applied.Marked {
|
||||
// 应用期停在"标记态"(claim 未收尾)时,applied 回执说的就是那份标记体。
|
||||
claim, err := t.indexClaimBody(c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return indexBytesEqual(observed, claim), nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (t *ProductionLibraryTarget) restoreIndexClaimBody(c libraryrelease.IndexWrite) (json.RawMessage, error) {
|
||||
var body map[string]json.RawMessage
|
||||
if json.Unmarshal(c.Before, &body) != nil || body == nil {
|
||||
@@ -52,15 +79,6 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
|
||||
if err = t.saveIndexEffect(c, "restore-mark-intent", &applied); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt, err := t.mutatePlannedIndex(ctx, c, body, applied.SeqNo, applied.PrimaryTerm, len(c.After) == 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
receipt.Marked = true
|
||||
if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt}
|
||||
}
|
||||
finalizing, err := t.loadIndexEffect(c, "restore-finalize-intent")
|
||||
if err != nil {
|
||||
@@ -73,6 +91,35 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !indexBytesEqual(observed, body) {
|
||||
// The marker is absent: either the intent never reached the index (a
|
||||
// deterministic refusal before dispatch) or its write outcome is unknown.
|
||||
// Only an unchanged owned generation separates the two; an unknown write
|
||||
// stays held (B19) and is never adopted by appearance.
|
||||
unsent, err := t.restorationProvablyUnsent(observed, c, applied)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !unsent {
|
||||
return errors.New("unknown restoration lacks exact operation marker/content/CAS; held")
|
||||
}
|
||||
if err := t.refuseRestoreWrite(c); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt, err := t.mutatePlannedIndex(ctx, c, body, applied.SeqNo, applied.PrimaryTerm, len(c.After) == 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
receipt.Marked = true
|
||||
if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt}
|
||||
observed, err = t.observePlannedIndex(ctx, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !indexBytesEqual(observed, body) || observed.SeqNo == nil || observed.PrimaryTerm == nil {
|
||||
return errors.New("unknown restoration lacks exact operation marker/content/CAS; held")
|
||||
}
|
||||
@@ -92,6 +139,9 @@ func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context,
|
||||
if err = t.saveIndexEffect(c, "restore-finalize-intent", claimed.Receipt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.refuseRestoreWrite(c); err != nil {
|
||||
return err
|
||||
}
|
||||
receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, claimed.Receipt.SeqNo, claimed.Receipt.PrimaryTerm, false)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -102,6 +102,10 @@ type ProductionLibraryTarget struct {
|
||||
preserveChunkSequence bool
|
||||
activePlan *libraryrelease.WriteSet
|
||||
activeSnapshot string
|
||||
// restoreMarkerFault 是一次性控制流故障注入点(生产恒为 nil):它在恢复标记的
|
||||
// 持久意图落盘之后、网络派发之前返回错误,用来区分"边界上确定没写进去"与
|
||||
// "写入结果未知"——前者重试时只凭世代证据重做,后者仍按 B19 永久 held。
|
||||
restoreMarkerFault func(libraryrelease.IndexWrite) error
|
||||
}
|
||||
|
||||
// resolveStorageBackend 在目标库里找本次写入的存储后端。找不到就报错:宁可不写,
|
||||
|
||||
Reference in New Issue
Block a user