98294b9583
build / build (push) Failing after 24s
why:审计 19 项(12 P1 / 7 P2)来自同一批根因,逐点试错会把协议撕成十几份:
· 意图与执行混为一谈(F01/F15):`library.sync:<scope>:<快照>` 是终身幂等键,
A→B→A 的第三次发布拿到第一次的历史成功作业,什么都不做;
· 版本行既是实体又是事件(F02/F09):主键只由 (kind,scope,快照) 决定、写入是
upsert,第二个作业覆盖 `source_job_id`/`evidence`/`created_at`;
· 接管没有代次(F13):两个 worker 都能认领同一 pending 作业,失租的旧执行还能
写业务状态、释放别人的维护锁;
· 早退与终态没有统一政策(F03/F04/F05/F06):半写窗口的前置检查失败会重新开门、
服务起不来也照样开入口、journal 终态后收尾不可重建、恢复成功不回写 current;
· 数据身份与准入各写一套(F07/F08/F14/F17):接管丢了复用语义、全库搬运不改
location_hash、通用入口可造 library.* 作业、目标模型从没被问过;
· 存储/容量/UI/CI 判据各自为政(F10/F11/F12/F16/F18/F19)。
what:按批准的一套协议集中实施(详见 handoffs/release-consolidated-implementation/
current-contract.md):
· operation(一次明确意图)/ attempt(epoch 代次)/ 不可变版本事件 / current 指针;
治理库迁移 76,只动治理库,不碰 WeKnora Schema/源码/镜像;
· 所有业务状态、阶段、版本事件、维护锁写入带 epoch 守卫(失租代次写不进去);
心跳失租立刻取消 handler;
· 半写窗口早退保持关闭;服务未全部健康绝不开入口;journal 终态可从持久事实重建
收尾;恢复成功登记 recovery 事件并推进 current;
· worker 在动维护前核对业务行/kind/scope/队列关联;通用入口拒绝 library.*;
· 全库搬运重算 location_hash 并与文档级导入共用实现;接管消费写集合的复用决定;
发布前核目标实时模型契约、容量门禁前置;
· 存储保护闭包改为全量依赖、类型只认显式登记、保护未知则不出可删候选;
· 页面核 HTTP 状态与结构、恢复纳入轮询、Wiki 差异覆盖标题/摘要/父级。
how:`make verify` rc=0(16 项必需检查全绿,含 go test 全量与前端 485 例);
`go vet ./...` rc=0;`npx tsc -p tsconfig.json --noEmit` rc=0;逐项负例见各包测试
(F03/F04/F08/F12/F13/F14/F16/F18/F19 都有先红后绿记录)。
未执行:治理库迁移到任何已有库、真实 PG/ES/S3/WeKnora 集成、故障注入、部署。
307 lines
14 KiB
Go
307 lines
14 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"os"
|
||
"path/filepath"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/libraryrelease"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
// 现场(2026-10-30,job librestore-internal-1793321916382871758-retry):
|
||
// 第一次中断后接管复用了同一份预备份,BackupPrepared 用一份**新造的**证据
|
||
// 覆盖了已持久化的 writeset_*,于是 verify_effective 的 writeset_digest 从
|
||
// fddd98cf… 变成了空;第二次中断再接管时 loadWriteSetForJob 已经读不回记录。
|
||
// 预备份本身没变(快照 id / manifest / 首次 created_at 都不变),丢的只有台账。
|
||
//
|
||
// 这些用例必须跑在真实 PostgreSQL 上:丢证据的机制是 RecordLibraryVersion 的
|
||
// `evidence = EXCLUDED.evidence`(整份覆盖),而"同一身份归谁"也是存储层语义
|
||
// (id 由 kind+scope+快照决定、upsert 会改写 source_job_id)。假 store 造不出这两件事。
|
||
|
||
// resumeLedgerFixture 是本组用例的账本夹具:一个作业、一份预备份、一份写集合。
|
||
type resumeLedgerFixture struct {
|
||
pool *pgxpool.Pool
|
||
scope string
|
||
jobID string
|
||
backup string
|
||
target string
|
||
params string
|
||
root string
|
||
store *postgresstore.Store
|
||
record libraryStageRecorder
|
||
writeSet libraryrelease.WriteSet
|
||
}
|
||
|
||
func newResumeLedgerFixture(t *testing.T, suffix string) *resumeLedgerFixture {
|
||
t.Helper()
|
||
store, pool := openLedgerStore(t)
|
||
ctx := context.Background()
|
||
now := time.Now().UTC()
|
||
f := &resumeLedgerFixture{
|
||
pool: pool,
|
||
scope: "internal",
|
||
jobID: "librestore-internal-ws-" + suffix,
|
||
backup: "backup-snap-ws-" + suffix,
|
||
target: "target-snap-ws-" + suffix,
|
||
params: "params-ws-" + suffix,
|
||
root: t.TempDir(),
|
||
store: store,
|
||
}
|
||
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{ID: f.jobID, Scope: f.scope,
|
||
Kind: governance.LibraryJobRestore, TargetSnapshotID: f.target, RequestDigest: f.params,
|
||
IdempotencyKey: "library.restore:internal:" + suffix}, governance.SyncStages(), now); err != nil {
|
||
t.Fatalf("create library job: %v", err)
|
||
}
|
||
f.record = libraryStageRecorder{store: store, jobID: f.jobID, scope: f.scope, mappingDigest: "mapping-live"}
|
||
// 首轮:真实预备份字节(4096)与写集合都按生产顺序落库/落盘。
|
||
if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-" + f.backup,
|
||
SnapshotID: f.backup, Bytes: 4096}); err != nil {
|
||
t.Fatalf("首轮登记预备份:%v", err)
|
||
}
|
||
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: f.jobID, Target: f.scope,
|
||
TenantID: 10001, KnowledgeBaseID: "kb-resume", BackupSnapshotID: f.backup,
|
||
TargetSnapshotID: f.target, CreatedAt: now,
|
||
Rows: []libraryrelease.WriteSetRow{{Table: "resources", ID: "res-1"}, {Table: "chunks", ID: "chunk-1"}},
|
||
Universe: map[string]libraryrelease.TableFingerprint{
|
||
"resources": {Count: 1, Digest: "u"}}}
|
||
if err := set.Seal(); err != nil {
|
||
t.Fatalf("seal write set: %v", err)
|
||
}
|
||
if _, err := v071.PersistWriteSet(f.root, &set); err != nil {
|
||
t.Fatalf("persist write set: %v", err)
|
||
}
|
||
if err := f.record.WriteSetPlanned(ctx, set); err != nil {
|
||
t.Fatalf("登记写集合:%v", err)
|
||
}
|
||
f.writeSet = set
|
||
return f
|
||
}
|
||
|
||
// evidenceOf 直接读库,绕过被测代码:证据里到底还剩什么,由这里说了算。
|
||
func (f *resumeLedgerFixture) evidenceOf(t *testing.T, kind governance.LibraryVersionKind, snapshot string) map[string]any {
|
||
t.Helper()
|
||
var raw []byte
|
||
if err := f.pool.QueryRow(context.Background(), `SELECT evidence FROM library_versions
|
||
WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4`,
|
||
f.scope, string(kind), f.jobID, snapshot).Scan(&raw); err != nil {
|
||
t.Fatalf("read %s evidence: %v", kind, err)
|
||
}
|
||
var evidence map[string]any
|
||
if err := json.Unmarshal(raw, &evidence); err != nil {
|
||
t.Fatalf("decode %s evidence: %v", kind, err)
|
||
}
|
||
return evidence
|
||
}
|
||
|
||
func (f *resumeLedgerFixture) createdAtOf(t *testing.T, kind governance.LibraryVersionKind, snapshot string) time.Time {
|
||
t.Helper()
|
||
var at time.Time
|
||
if err := f.pool.QueryRow(context.Background(), `SELECT created_at FROM library_versions
|
||
WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4`,
|
||
f.scope, string(kind), f.jobID, snapshot).Scan(&at); err != nil {
|
||
t.Fatalf("read %s created_at: %v", kind, err)
|
||
}
|
||
return at.UTC()
|
||
}
|
||
|
||
// interruptedTarget 模拟"写某一类数据时进程被杀":写入阶段失败,恢复链路照常执行。
|
||
type interruptedTarget struct {
|
||
failAt libraryrelease.Stage
|
||
applied int
|
||
}
|
||
|
||
func (t *interruptedTarget) Current(context.Context) (libraryrelease.CurrentState, error) {
|
||
return libraryrelease.CurrentState{Counts: map[string]int64{"resources": 1},
|
||
Digests: map[string]string{"resources": "u"}}, nil
|
||
}
|
||
|
||
func (t *interruptedTarget) Apply(_ context.Context, stage libraryrelease.Stage, _ string) error {
|
||
t.applied++
|
||
if stage == t.failAt {
|
||
return errors.New("simulated worker kill during apply")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (t *interruptedTarget) Verify(context.Context, libraryrelease.Expectation) error { return nil }
|
||
func (t *interruptedTarget) Restore(context.Context, libraryrelease.BackupRef) error { return nil }
|
||
|
||
// completeTarget 是一条能跑完的链路:用来证明最后一次真实恢复仍带着同一份写集合,
|
||
// 并且生效版本证据里是真实字节(不是复用路径上被抹成 0 的那个)。
|
||
type completeTarget struct{ applied int }
|
||
|
||
func (t *completeTarget) Current(context.Context) (libraryrelease.CurrentState, error) {
|
||
return libraryrelease.CurrentState{Counts: map[string]int64{"resources": 1},
|
||
Digests: map[string]string{"resources": "u"}}, nil
|
||
}
|
||
func (t *completeTarget) Apply(context.Context, libraryrelease.Stage, string) error {
|
||
t.applied++
|
||
return nil
|
||
}
|
||
func (t *completeTarget) Verify(context.Context, libraryrelease.Expectation) error { return nil }
|
||
func (t *completeTarget) Restore(context.Context, libraryrelease.BackupRef) error { return nil }
|
||
|
||
// takeoverRequest 是接管时用的发布请求:身份与首轮完全一致。
|
||
func (f *resumeLedgerFixture) takeoverRequest(now time.Time) libraryrelease.Request {
|
||
return libraryrelease.Request{JobID: f.jobID, Target: f.scope, SnapshotID: f.target,
|
||
ParamsDigest: f.params, Now: now,
|
||
Grant: libraryrelease.Grant{Target: f.scope, SnapshotID: f.target, ParamsDigest: f.params,
|
||
ExpiresAt: now.Add(time.Hour), AuthorizedBy: "tester"}}
|
||
}
|
||
|
||
// takeoverBackups 按生产代码的方式装配接管来源:先按身份取回写集合与真实字节,
|
||
// 再交给 ledgerBackupStore。缺任何一样都必须失败,不许降级成"按目标现状推断"。
|
||
func (f *resumeLedgerFixture) takeoverBackups(t *testing.T, store *postgresstore.Store) libraryrelease.BackupStore {
|
||
t.Helper()
|
||
ctx := context.Background()
|
||
set, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup)
|
||
if err != nil {
|
||
t.Fatalf("接管读不回写集合(作业 %s,预备份 %s):%v", f.jobID, f.backup, err)
|
||
}
|
||
bytes, err := prebackupBytesOf(ctx, store, f.scope, f.jobID, f.backup)
|
||
if err != nil {
|
||
t.Fatalf("接管读不回预备份字节:%v", err)
|
||
}
|
||
if bytes != 4096 {
|
||
t.Fatalf("接管拿到的预备份字节 = %d, want 4096(复用不许把真实字节抹成 0)", bytes)
|
||
}
|
||
return ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + f.backup,
|
||
SnapshotID: f.backup, Bytes: bytes}, write: set}
|
||
}
|
||
|
||
func TestTakeoverKeepsWriteSetEvidenceAcrossRepeatedInterruptions(t *testing.T) {
|
||
f := newResumeLedgerFixture(t, "repeat")
|
||
store := f.store
|
||
ctx := context.Background()
|
||
firstCreatedAt := f.createdAtOf(t, governance.LibraryVersionProductionPrebackup, f.backup)
|
||
|
||
// 两次"写到一半被杀 + 接管":每次接管都会复用同一份预备份并重新登记证据。
|
||
for round := 1; round <= 2; round++ {
|
||
backups := f.takeoverBackups(t, store)
|
||
now := time.Now().UTC()
|
||
outcome := libraryrelease.Run(ctx, f.takeoverRequest(now),
|
||
&interruptedTarget{failAt: libraryrelease.Stage("database")}, backups, f.record)
|
||
if outcome.State != "rolled_back" {
|
||
t.Fatalf("第 %d 次中断的结局 = %q(%s / %s),want rolled_back",
|
||
round, outcome.State, outcome.Error, outcome.Rollback)
|
||
}
|
||
// 每次中断之后,下一次接管/恢复都必须还能按身份取回写集合。
|
||
set, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup)
|
||
if err != nil {
|
||
t.Fatalf("第 %d 次接管后写集合丢失(正是本缺陷):%v", round, err)
|
||
}
|
||
if set.Digest != f.writeSet.Digest {
|
||
t.Fatalf("第 %d 次接管后写集合摘要 = %s, want %s", round, set.Digest, f.writeSet.Digest)
|
||
}
|
||
}
|
||
|
||
// 证据逐项核对:写集合锚点、真实字节、作业/预备份身份都必须还在。
|
||
prebackup := f.evidenceOf(t, governance.LibraryVersionProductionPrebackup, f.backup)
|
||
for key, want := range map[string]any{
|
||
"writeset_digest": f.writeSet.Digest,
|
||
"writeset_rows": float64(2),
|
||
"job": f.jobID,
|
||
"backup_snapshot": f.backup,
|
||
"backup_bytes": float64(4096),
|
||
"index_mapping_sha256": "mapping-live",
|
||
"backup_reused": true,
|
||
} {
|
||
if got := prebackup[key]; got != want {
|
||
t.Fatalf("复用后证据 %s = %v, want %v(整份覆盖就是本缺陷)", key, got, want)
|
||
}
|
||
}
|
||
if got := f.createdAtOf(t, governance.LibraryVersionProductionPrebackup, f.backup); !got.Equal(firstCreatedAt) {
|
||
t.Fatalf("复用后预备份登记时刻 = %s, want %s(备份什么时候拍的被洗掉了)", got, firstCreatedAt)
|
||
}
|
||
|
||
// 第三次不中断:完整跑通,证明最后一次真实恢复仍按同一份写集合执行。
|
||
final := &completeTarget{}
|
||
outcome := libraryrelease.Run(ctx, f.takeoverRequest(time.Now().UTC()), final, f.takeoverBackups(t, store), f.record)
|
||
if outcome.State != "succeeded" {
|
||
t.Fatalf("最终恢复 state=%q error=%s rollback=%s", outcome.State, outcome.Error, outcome.Rollback)
|
||
}
|
||
if final.applied != len(libraryrelease.Stages()) {
|
||
t.Fatalf("四类写入没有全部执行:applied=%d", final.applied)
|
||
}
|
||
// 生效版本证据由处理器用 outcome.Backup 构造(这里不重复实现那一层),
|
||
// 所以断言的是它拿到的输入:复用路径必须带回真实字节与同一份预备份身份。
|
||
if outcome.Backup.Bytes != 4096 {
|
||
t.Fatalf("复用路径交回的备份字节 = %d, want 4096(处理器会把 0 写进生效版本证据)",
|
||
outcome.Backup.Bytes)
|
||
}
|
||
if !outcome.Backup.Reused || outcome.Backup.SnapshotID != f.backup {
|
||
t.Fatalf("复用路径交回的预备份身份不对:%+v", outcome.Backup)
|
||
}
|
||
effective := effectiveEvidence(outcome.Backup, "restore", "mapping-live")
|
||
if effective["backup_bytes"] != int64(4096) || effective["backup_snapshot"] != f.backup {
|
||
t.Fatalf("生效版本证据 = %v, want backup_bytes 4096 + backup_snapshot %s", effective, f.backup)
|
||
}
|
||
}
|
||
|
||
// 负例:复用声明与已持久化证据对不上时,必须 fail closed —— 既不许凭空登记,
|
||
// 也不许借用别的作业/别的预备份的记录。
|
||
func TestTakeoverLedgerRefusesForeignOrMissingBackupEvidence(t *testing.T) {
|
||
f := newResumeLedgerFixture(t, "negatives")
|
||
store := f.store
|
||
ctx := context.Background()
|
||
|
||
// 声称复用一份从没登记过的预备份:没有证据可合并 ⇒ 拒绝。
|
||
if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-unknown",
|
||
SnapshotID: "backup-snap-never-registered", Bytes: 1, Reused: true}); err == nil {
|
||
t.Fatal("复用未登记的预备份必须失败(否则就是把复用洗成新备份)")
|
||
}
|
||
|
||
// 同一份预备份被别的作业占用:LibraryVersionByRef 必须报身份冲突,而不是当没有。
|
||
if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-" + f.backup,
|
||
SnapshotID: f.backup, Bytes: 4096, Reused: true}); err != nil {
|
||
t.Fatalf("本方复用同一份预备份应当成功:%v", err)
|
||
}
|
||
|
||
// 字节数对不上(本次声称 2048,已登记 4096)⇒ 拒绝:宁可保持维护,也不许写错基线。
|
||
if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-" + f.backup,
|
||
SnapshotID: f.backup, Bytes: 2048, Reused: true}); err == nil {
|
||
t.Fatal("复用预备份字节数不一致时必须失败")
|
||
}
|
||
|
||
// 写集合被篡改:文件内容改一个字节,摘要就对不上 ⇒ 拒绝恢复。
|
||
path := filepath.Join(f.root, "writesets", f.writeSet.Digest+".json")
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatalf("read write set file: %v", err)
|
||
}
|
||
tampered := []byte(string(raw[:len(raw)-2]) + "X}")
|
||
// 落盘的写集合是 0444(只读台账):篡改必须先放开权限,这也顺带证明它默认不可写。
|
||
if err := os.Chmod(path, 0600); err != nil {
|
||
t.Fatalf("chmod write set: %v", err)
|
||
}
|
||
if err := os.WriteFile(path, tampered, 0600); err != nil {
|
||
t.Fatalf("tamper write set: %v", err)
|
||
}
|
||
if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup); err == nil {
|
||
t.Fatal("写集合被篡改后必须拒绝取回")
|
||
}
|
||
if err := os.WriteFile(path, raw, 0600); err != nil {
|
||
t.Fatalf("restore write set: %v", err)
|
||
}
|
||
if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup); err != nil {
|
||
t.Fatalf("恢复原文件后应能取回:%v", err)
|
||
}
|
||
|
||
// 换了预备份身份 / 换了作业:都不许借到这份记录。
|
||
if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, "backup-snap-other"); err == nil {
|
||
t.Fatal("不同预备份必须拒绝")
|
||
}
|
||
if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, "librestore-internal-ws-other", f.backup); err == nil {
|
||
t.Fatal("不同作业必须拒绝")
|
||
}
|
||
}
|