513aa9e6f5
build / build (push) Has been cancelled
为什么:Astra 复核证实五组实现缺陷——正式装配缺 scope/快照且来源与 before-image 混用;激活撤销不在 CAS 事务内、lease 用事务起始时间;激活只接受任意字符串且撤销可 被 SQL 重置/删除;准入忽略绑定冲突与 prepared 矛盾、收尾只看当前 plan;prepared 已 提交但 mark-intent 未落盘时被误判 legacy。 改了什么: - 绑定来自原锚点(scope/来源快照/backup/实例/目标指纹/索引 UUID/恢复投递),每次 重建都实时重读目标身份并与冻结值比较; - activation 行锁与 CAS 同事务,lease 用 clock_timestamp(),同事务核对 job/operation/ attempt/维护窗口/恢复 queue 与 operation,同 effect 换 plan 摘要拒绝; - 激活消费带摘要的证据文件(writer 退出+排空、目标/治理库身份、有效期),撤销终局, down 在 activation 有行时拒绝; - 前置/收尾共用权威窗口全集,绑定冲突与 prepared 矛盾零写 held,合法 dispatching+ 精确 marker 仍可确认; - prepared 无 mark-intent 仅在绑定一致且无矛盾时补同内容意图,旧无台账意图仍 held。 验证:真 PG 锁等待覆盖撤销先胜/CAS 先胜/等锁跨租约/错锚点/plan 别名/恢复投递; COMMIT 已到服务端但回包丢失与未决事务分别读回;真 ES+PG+对象存储的 DS01 用例全部 通过;go test ./... 全绿。未迁移正式库、未部署、未激活。
367 lines
12 KiB
Go
367 lines
12 KiB
Go
package v071
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"knowledge-governance/internal/libraryrelease"
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
)
|
|
|
|
type indexReceipt struct {
|
|
Index string `json:"_index"`
|
|
ID string `json:"_id"`
|
|
Result string `json:"result"`
|
|
SeqNo *int64 `json:"_seq_no"`
|
|
PrimaryTerm *int64 `json:"_primary_term"`
|
|
Marked bool `json:"marked,omitempty"`
|
|
}
|
|
type indexEffect struct {
|
|
Plan string `json:"plan"`
|
|
Change libraryrelease.IndexWrite `json:"change"`
|
|
Receipt *indexReceipt `json:"receipt,omitempty"`
|
|
}
|
|
|
|
// beginRestoreMarker 在第一次恢复写入派发之前,把恢复标记意图原子落盘。
|
|
// 证据只描述"我们要写什么",不记录任何用来证明"没写过"的计数:
|
|
// ES 的写计数器是分片生命周期内的统计量(分片迁移/节点替换会清零),
|
|
// 不是不可变事务日志,不能作为放行写入的授权依据。
|
|
func (t *ProductionLibraryTarget) beginRestoreMarker(ctx context.Context, c libraryrelease.IndexWrite, applied indexReceipt) error {
|
|
if t.restoreIntentFault != nil {
|
|
fault := t.restoreIntentFault
|
|
t.restoreIntentFault = nil
|
|
if err := fault(c); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
raw, err := json.Marshal(indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &applied})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return persistImmutableJSON(t.indexEffectPath(c, "restore-mark-intent"), raw)
|
|
}
|
|
|
|
func (t *ProductionLibraryTarget) indexEffectPath(c libraryrelease.IndexWrite, phase string) string {
|
|
sum := sha256.Sum256([]byte(c.Index + "\x00" + c.ID))
|
|
return filepath.Join(t.config.ArchiveRoot, "effects", t.activePlan.Digest, "index", hex.EncodeToString(sum[:]), phase+".json")
|
|
}
|
|
func (t *ProductionLibraryTarget) saveIndexEffect(c libraryrelease.IndexWrite, phase string, r *indexReceipt) error {
|
|
raw, err := json.Marshal(indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: r})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return persistImmutableJSON(t.indexEffectPath(c, phase), raw)
|
|
}
|
|
func (t *ProductionLibraryTarget) loadIndexEffect(c libraryrelease.IndexWrite, phase string) (*indexEffect, error) {
|
|
path := t.indexEffectPath(c, phase)
|
|
exists, err := effectExists(path)
|
|
if err != nil || !exists {
|
|
return nil, err
|
|
}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var e indexEffect
|
|
if json.Unmarshal(raw, &e) != nil {
|
|
return nil, errors.New("invalid index effect evidence")
|
|
}
|
|
want, _ := json.Marshal(c)
|
|
got, _ := json.Marshal(e.Change)
|
|
if e.Plan != t.activePlan.Digest || !bytes.Equal(want, got) {
|
|
return nil, errors.New("index effect binding differs from immutable plan")
|
|
}
|
|
if e.Receipt != nil && (e.Receipt.Index != c.Index || e.Receipt.ID != c.ID || e.Receipt.SeqNo == nil || e.Receipt.PrimaryTerm == nil) {
|
|
return nil, errors.New("incomplete index receipt")
|
|
}
|
|
return &e, nil
|
|
}
|
|
func (t *ProductionLibraryTarget) observePlannedIndex(ctx context.Context, c libraryrelease.IndexWrite) (indexWriteGuard, error) {
|
|
guards, err := t.search.indexWriteGuards(ctx, []librarysnapshot.IndexDocument{{ID: c.ID}}, SearchIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID})
|
|
if err != nil {
|
|
return indexWriteGuard{}, err
|
|
}
|
|
return guards[c.ID], nil
|
|
}
|
|
func indexBytesEqual(g indexWriteGuard, want json.RawMessage) bool {
|
|
if len(want) == 0 {
|
|
return !g.Found
|
|
}
|
|
if !g.Found {
|
|
return false
|
|
}
|
|
raw, err := canonicalJSON(g.Source)
|
|
return err == nil && bytes.Equal(raw, want)
|
|
}
|
|
func sameIndexGeneration(g indexWriteGuard, seq, term *int64) bool {
|
|
return g.Found && g.SeqNo != nil && g.PrimaryTerm != nil && seq != nil && term != nil && *g.SeqNo == *seq && *g.PrimaryTerm == *term
|
|
}
|
|
|
|
// refuseRestoreWrite 是恢复写入的派发边界(生产恒为 nil):验收用例在这里制造
|
|
// "边界上确定失败"——方法在派发前返回错误,真实 ES 一个字节都收不到,这与断线、
|
|
// 响应丢失(写入结果未知,按 B19 必须 held)是两件不同的事。
|
|
func (t *ProductionLibraryTarget) refuseRestoreWrite(c libraryrelease.IndexWrite) error {
|
|
if t.restoreMarkerFault == nil {
|
|
return nil
|
|
}
|
|
return t.restoreMarkerFault(c)
|
|
}
|
|
|
|
// Each network mutation has an immutable intent before send and a per-ID
|
|
// server generation receipt. A lost response is UNKNOWN, never "did not write".
|
|
// No retry is issued for an unresolved intent; downstream ownership fencing is
|
|
// required before unknown operations can be conclusively reconciled.
|
|
func (t *ProductionLibraryTarget) applyPlannedIndex(ctx context.Context) error {
|
|
for _, c := range t.activePlan.Index {
|
|
if bytes.Equal(c.Before, c.After) {
|
|
continue
|
|
}
|
|
applied, err := t.loadIndexEffect(c, "applied")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if applied != nil {
|
|
if applied.Receipt == nil {
|
|
return errors.New("index receipt missing")
|
|
}
|
|
g, err := t.observePlannedIndex(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !indexBytesEqual(g, c.After) || (len(c.After) > 0 && !sameIndexGeneration(g, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm)) {
|
|
return errors.New("index changed after confirmed apply")
|
|
}
|
|
continue
|
|
}
|
|
intent, err := t.loadIndexEffect(c, "apply-intent")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if intent != nil {
|
|
if _, err := t.reconcileIndexClaim(ctx, c); err != nil {
|
|
return err
|
|
}
|
|
receipt, err := t.finalizeIndexClaim(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := t.saveIndexEffect(c, "applied", &receipt); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
g, err := t.observePlannedIndex(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !indexBytesEqual(g, c.Before) || (len(c.Before) > 0 && !sameIndexGeneration(g, c.SeqNo, c.PrimaryTerm)) {
|
|
return errors.New("index changed since frozen plan")
|
|
}
|
|
if err = t.saveIndexEffect(c, "apply-intent", nil); err != nil {
|
|
return err
|
|
}
|
|
receipt, err := t.applyIndexClaim(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = t.saveIndexEffect(c, "applied", &receipt); err != nil {
|
|
return err
|
|
}
|
|
g, err = t.observePlannedIndex(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !indexBytesEqual(g, c.After) || (len(c.After) > 0 && !sameIndexGeneration(g, receipt.SeqNo, receipt.PrimaryTerm)) {
|
|
return errors.New("index readback differs from receipt")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func (t *ProductionLibraryTarget) indexMutation(c libraryrelease.IndexWrite, body json.RawMessage, seq, term *int64, create bool) (string, string, any, error) {
|
|
method := http.MethodPut
|
|
endpoint := "_doc"
|
|
if create {
|
|
endpoint = "_create"
|
|
}
|
|
if len(body) == 0 {
|
|
method = http.MethodDelete
|
|
}
|
|
query := url.Values{"refresh": {"true"}}
|
|
if !create {
|
|
if seq == nil || term == nil {
|
|
return "", "", nil, errors.New("index mutation lacks CAS")
|
|
}
|
|
query.Set("if_seq_no", fmt.Sprint(*seq))
|
|
query.Set("if_primary_term", fmt.Sprint(*term))
|
|
}
|
|
path := "/" + url.PathEscape(c.Index) + "/" + endpoint + "/" + url.PathEscape(c.ID) + "?" + query.Encode()
|
|
var payload any
|
|
if len(body) > 0 {
|
|
payload = body
|
|
}
|
|
return method, path, payload, nil
|
|
}
|
|
|
|
func validateIndexReceipt(c libraryrelease.IndexWrite, hasBody bool, receipt indexReceipt) error {
|
|
if receipt.ID != c.ID || receipt.Index != c.Index || receipt.SeqNo == nil || receipt.PrimaryTerm == nil {
|
|
return errors.New("index mutation receipt incomplete")
|
|
}
|
|
if !hasBody && receipt.Result != "deleted" || hasBody && receipt.Result != "created" && receipt.Result != "updated" {
|
|
return errors.New("index mutation receipt not confirmed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, seq, term *int64, create bool) (indexReceipt, error) {
|
|
method, path, payload, err := t.indexMutation(c, body, seq, term, create)
|
|
if err != nil {
|
|
return indexReceipt{}, err
|
|
}
|
|
var receipt indexReceipt
|
|
if err := t.search.requestJSON(ctx, method, path, payload, &receipt); err != nil {
|
|
return receipt, fmt.Errorf("index effect outcome unknown: %w", err)
|
|
}
|
|
if err := validateIndexReceipt(c, len(body) > 0, receipt); err != nil {
|
|
return receipt, err
|
|
}
|
|
return receipt, nil
|
|
}
|
|
func (t *ProductionLibraryTarget) restorePlannedIndex(ctx context.Context) error {
|
|
for _, c := range t.activePlan.Index {
|
|
intent, err := t.loadIndexEffect(c, "apply-intent")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if intent == nil {
|
|
continue
|
|
}
|
|
// Completed compensation is read back before attempting to reconcile an
|
|
// old marked apply; the marker was intentionally removed by restoration.
|
|
done, err := t.loadIndexEffect(c, "restored")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if done != nil {
|
|
g, err := t.observePlannedIndex(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !indexBytesEqual(g, c.Before) || (len(c.Before) > 0 && (done.Receipt == nil || !sameIndexGeneration(g, done.Receipt.SeqNo, done.Receipt.PrimaryTerm))) {
|
|
return errors.New("restored index changed")
|
|
}
|
|
continue
|
|
}
|
|
if reconciled, err := t.reconcileIndexCompensationDelete(ctx, c); err != nil {
|
|
return err
|
|
} else if reconciled {
|
|
continue
|
|
}
|
|
applied, err := t.loadIndexEffect(c, "applied")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if applied == nil {
|
|
receipt, err := t.reconcileIndexClaim(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
applied = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt}
|
|
}
|
|
if applied.Receipt == nil {
|
|
return errors.New("missing index compensation receipt")
|
|
}
|
|
g, err := t.observePlannedIndex(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
restoring, err := t.loadIndexEffect(c, "restore-intent")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if restoring != nil {
|
|
if len(c.Before) > 0 {
|
|
if err := t.restoreClaimedIndexBefore(ctx, c, *applied.Receipt, false); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
// 删除型补偿:删除落地必然移除文档并推进世代;文档仍是 applied 结果、
|
|
// 世代仍是 applied 回执,才证明那次 DELETE 没有落地,可按同一 CAS 重放。
|
|
expected := c.After
|
|
if applied.Receipt.Marked {
|
|
expected, err = t.indexClaimBody(c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if !indexBytesEqual(g, expected) || !sameIndexGeneration(g, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm) {
|
|
return errors.New("unknown index compensation requires reconciliation")
|
|
}
|
|
if err := t.refuseRestoreWrite(c); err != nil {
|
|
return err
|
|
}
|
|
receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = t.saveIndexEffect(c, "restored", &receipt); err != nil {
|
|
return err
|
|
}
|
|
g, err = t.observePlannedIndex(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !indexBytesEqual(g, c.Before) {
|
|
return errors.New("index compensation readback failed")
|
|
}
|
|
continue
|
|
}
|
|
expected := c.After
|
|
if applied.Receipt.Marked {
|
|
expected, err = t.indexClaimBody(c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if !indexBytesEqual(g, expected) || (len(expected) > 0 && !sameIndexGeneration(g, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm)) {
|
|
return errors.New("another writer changed index; compensation refused")
|
|
}
|
|
if err = t.saveIndexEffect(c, "restore-intent", applied.Receipt); err != nil {
|
|
return err
|
|
}
|
|
if len(c.Before) > 0 {
|
|
if err := t.restoreClaimedIndexBefore(ctx, c, *applied.Receipt, true); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
if err := t.refuseRestoreWrite(c); err != nil {
|
|
return err
|
|
}
|
|
receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm, len(c.After) == 0)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = t.saveIndexEffect(c, "restored", &receipt); err != nil {
|
|
return err
|
|
}
|
|
g, err = t.observePlannedIndex(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !indexBytesEqual(g, c.Before) {
|
|
return errors.New("index compensation readback failed")
|
|
}
|
|
}
|
|
return nil
|
|
}
|