Files
knowledge-governance/cmd/worker/library_writeset_resume_integration_test.go
T
姚章浩 98294b9583
build / build (push) Failing after 24s
fix(library): 统一执行身份、终态收尾、准入与数据身份
why:审计 19 项(12 P1 / 7 P2)来自同一批根因,逐点试错会把协议撕成十几份:
  · 意图与执行混为一谈(F01/F15):`library.sync:<scope>:<快照>` 是终身幂等键,
    A→B→A 的第三次发布拿到第一次的历史成功作业,什么都不做;
  · 版本行既是实体又是事件(F02/F09):主键只由 (kind,scope,快照) 决定、写入是
    upsert,第二个作业覆盖 `source_job_id`/`evidence`/`created_at`;
  · 接管没有代次(F13):两个 worker 都能认领同一 pending 作业,失租的旧执行还能
    写业务状态、释放别人的维护锁;
  · 早退与终态没有统一政策(F03/F04/F05/F06):半写窗口的前置检查失败会重新开门、
    服务起不来也照样开入口、journal 终态后收尾不可重建、恢复成功不回写 current;
  · 数据身份与准入各写一套(F07/F08/F14/F17):接管丢了复用语义、全库搬运不改
    location_hash、通用入口可造 library.* 作业、目标模型从没被问过;
  · 存储/容量/UI/CI 判据各自为政(F10/F11/F12/F16/F18/F19)。

what:按批准的一套协议集中实施(详见 handoffs/release-consolidated-implementation/
current-contract.md):
  · operation(一次明确意图)/ attempt(epoch 代次)/ 不可变版本事件 / current 指针;
    治理库迁移 76,只动治理库,不碰 WeKnora Schema/源码/镜像;
  · 所有业务状态、阶段、版本事件、维护锁写入带 epoch 守卫(失租代次写不进去);
    心跳失租立刻取消 handler;
  · 半写窗口早退保持关闭;服务未全部健康绝不开入口;journal 终态可从持久事实重建
    收尾;恢复成功登记 recovery 事件并推进 current;
  · worker 在动维护前核对业务行/kind/scope/队列关联;通用入口拒绝 library.*;
  · 全库搬运重算 location_hash 并与文档级导入共用实现;接管消费写集合的复用决定;
    发布前核目标实时模型契约、容量门禁前置;
  · 存储保护闭包改为全量依赖、类型只认显式登记、保护未知则不出可删候选;
  · 页面核 HTTP 状态与结构、恢复纳入轮询、Wiki 差异覆盖标题/摘要/父级。

how:`make verify` rc=0(16 项必需检查全绿,含 go test 全量与前端 485 例);
`go vet ./...` rc=0;`npx tsc -p tsconfig.json --noEmit` rc=0;逐项负例见各包测试
(F03/F04/F08/F12/F13/F14/F16/F18/F19 都有先红后绿记录)。
未执行:治理库迁移到任何已有库、真实 PG/ES/S3/WeKnora 集成、故障注入、部署。
2026-10-30 16:14:54 +08:00

307 lines
14 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// 现场(2026-10-30,job librestore-internal-1793321916382871758-retry):
// 第一次中断后接管复用了同一份预备份,BackupPrepared 用一份**新造的**证据
// 覆盖了已持久化的 writeset_*,于是 verify_effective 的 writeset_digest 从
// fddd98cf… 变成了空;第二次中断再接管时 loadWriteSetForJob 已经读不回记录。
// 预备份本身没变(快照 id / manifest / 首次 created_at 都不变),丢的只有台账。
//
// 这些用例必须跑在真实 PostgreSQL 上:丢证据的机制是 RecordLibraryVersion 的
// `evidence = EXCLUDED.evidence`(整份覆盖),而"同一身份归谁"也是存储层语义
// (id 由 kind+scope+快照决定、upsert 会改写 source_job_id)。假 store 造不出这两件事。
// resumeLedgerFixture 是本组用例的账本夹具:一个作业、一份预备份、一份写集合。
type resumeLedgerFixture struct {
pool *pgxpool.Pool
scope string
jobID string
backup string
target string
params string
root string
store *postgresstore.Store
record libraryStageRecorder
writeSet libraryrelease.WriteSet
}
func newResumeLedgerFixture(t *testing.T, suffix string) *resumeLedgerFixture {
t.Helper()
store, pool := openLedgerStore(t)
ctx := context.Background()
now := time.Now().UTC()
f := &resumeLedgerFixture{
pool: pool,
scope: "internal",
jobID: "librestore-internal-ws-" + suffix,
backup: "backup-snap-ws-" + suffix,
target: "target-snap-ws-" + suffix,
params: "params-ws-" + suffix,
root: t.TempDir(),
store: store,
}
if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{ID: f.jobID, Scope: f.scope,
Kind: governance.LibraryJobRestore, TargetSnapshotID: f.target, RequestDigest: f.params,
IdempotencyKey: "library.restore:internal:" + suffix}, governance.SyncStages(), now); err != nil {
t.Fatalf("create library job: %v", err)
}
f.record = libraryStageRecorder{store: store, jobID: f.jobID, scope: f.scope, mappingDigest: "mapping-live"}
// 首轮:真实预备份字节(4096)与写集合都按生产顺序落库/落盘。
if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-" + f.backup,
SnapshotID: f.backup, Bytes: 4096}); err != nil {
t.Fatalf("首轮登记预备份:%v", err)
}
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: f.jobID, Target: f.scope,
TenantID: 10001, KnowledgeBaseID: "kb-resume", BackupSnapshotID: f.backup,
TargetSnapshotID: f.target, CreatedAt: now,
Rows: []libraryrelease.WriteSetRow{{Table: "resources", ID: "res-1"}, {Table: "chunks", ID: "chunk-1"}},
Universe: map[string]libraryrelease.TableFingerprint{
"resources": {Count: 1, Digest: "u"}}}
if err := set.Seal(); err != nil {
t.Fatalf("seal write set: %v", err)
}
if _, err := v071.PersistWriteSet(f.root, &set); err != nil {
t.Fatalf("persist write set: %v", err)
}
if err := f.record.WriteSetPlanned(ctx, set); err != nil {
t.Fatalf("登记写集合:%v", err)
}
f.writeSet = set
return f
}
// evidenceOf 直接读库,绕过被测代码:证据里到底还剩什么,由这里说了算。
func (f *resumeLedgerFixture) evidenceOf(t *testing.T, kind governance.LibraryVersionKind, snapshot string) map[string]any {
t.Helper()
var raw []byte
if err := f.pool.QueryRow(context.Background(), `SELECT evidence FROM library_versions
WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4`,
f.scope, string(kind), f.jobID, snapshot).Scan(&raw); err != nil {
t.Fatalf("read %s evidence: %v", kind, err)
}
var evidence map[string]any
if err := json.Unmarshal(raw, &evidence); err != nil {
t.Fatalf("decode %s evidence: %v", kind, err)
}
return evidence
}
func (f *resumeLedgerFixture) createdAtOf(t *testing.T, kind governance.LibraryVersionKind, snapshot string) time.Time {
t.Helper()
var at time.Time
if err := f.pool.QueryRow(context.Background(), `SELECT created_at FROM library_versions
WHERE scope=$1 AND kind=$2 AND source_job_id=$3 AND snapshot_id=$4`,
f.scope, string(kind), f.jobID, snapshot).Scan(&at); err != nil {
t.Fatalf("read %s created_at: %v", kind, err)
}
return at.UTC()
}
// interruptedTarget 模拟"写某一类数据时进程被杀":写入阶段失败,恢复链路照常执行。
type interruptedTarget struct {
failAt libraryrelease.Stage
applied int
}
func (t *interruptedTarget) Current(context.Context) (libraryrelease.CurrentState, error) {
return libraryrelease.CurrentState{Counts: map[string]int64{"resources": 1},
Digests: map[string]string{"resources": "u"}}, nil
}
func (t *interruptedTarget) Apply(_ context.Context, stage libraryrelease.Stage, _ string) error {
t.applied++
if stage == t.failAt {
return errors.New("simulated worker kill during apply")
}
return nil
}
func (t *interruptedTarget) Verify(context.Context, libraryrelease.Expectation) error { return nil }
func (t *interruptedTarget) Restore(context.Context, libraryrelease.BackupRef) error { return nil }
// completeTarget 是一条能跑完的链路:用来证明最后一次真实恢复仍带着同一份写集合,
// 并且生效版本证据里是真实字节(不是复用路径上被抹成 0 的那个)。
type completeTarget struct{ applied int }
func (t *completeTarget) Current(context.Context) (libraryrelease.CurrentState, error) {
return libraryrelease.CurrentState{Counts: map[string]int64{"resources": 1},
Digests: map[string]string{"resources": "u"}}, nil
}
func (t *completeTarget) Apply(context.Context, libraryrelease.Stage, string) error {
t.applied++
return nil
}
func (t *completeTarget) Verify(context.Context, libraryrelease.Expectation) error { return nil }
func (t *completeTarget) Restore(context.Context, libraryrelease.BackupRef) error { return nil }
// takeoverRequest 是接管时用的发布请求:身份与首轮完全一致。
func (f *resumeLedgerFixture) takeoverRequest(now time.Time) libraryrelease.Request {
return libraryrelease.Request{JobID: f.jobID, Target: f.scope, SnapshotID: f.target,
ParamsDigest: f.params, Now: now,
Grant: libraryrelease.Grant{Target: f.scope, SnapshotID: f.target, ParamsDigest: f.params,
ExpiresAt: now.Add(time.Hour), AuthorizedBy: "tester"}}
}
// takeoverBackups 按生产代码的方式装配接管来源:先按身份取回写集合与真实字节,
// 再交给 ledgerBackupStore。缺任何一样都必须失败,不许降级成"按目标现状推断"。
func (f *resumeLedgerFixture) takeoverBackups(t *testing.T, store *postgresstore.Store) libraryrelease.BackupStore {
t.Helper()
ctx := context.Background()
set, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup)
if err != nil {
t.Fatalf("接管读不回写集合(作业 %s,预备份 %s):%v", f.jobID, f.backup, err)
}
bytes, err := prebackupBytesOf(ctx, store, f.scope, f.jobID, f.backup)
if err != nil {
t.Fatalf("接管读不回预备份字节:%v", err)
}
if bytes != 4096 {
t.Fatalf("接管拿到的预备份字节 = %d, want 4096(复用不许把真实字节抹成 0)", bytes)
}
return ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + f.backup,
SnapshotID: f.backup, Bytes: bytes}, write: set}
}
func TestTakeoverKeepsWriteSetEvidenceAcrossRepeatedInterruptions(t *testing.T) {
f := newResumeLedgerFixture(t, "repeat")
store := f.store
ctx := context.Background()
firstCreatedAt := f.createdAtOf(t, governance.LibraryVersionProductionPrebackup, f.backup)
// 两次"写到一半被杀 + 接管":每次接管都会复用同一份预备份并重新登记证据。
for round := 1; round <= 2; round++ {
backups := f.takeoverBackups(t, store)
now := time.Now().UTC()
outcome := libraryrelease.Run(ctx, f.takeoverRequest(now),
&interruptedTarget{failAt: libraryrelease.Stage("database")}, backups, f.record)
if outcome.State != "rolled_back" {
t.Fatalf("第 %d 次中断的结局 = %q(%s / %s),want rolled_back",
round, outcome.State, outcome.Error, outcome.Rollback)
}
// 每次中断之后,下一次接管/恢复都必须还能按身份取回写集合。
set, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup)
if err != nil {
t.Fatalf("第 %d 次接管后写集合丢失(正是本缺陷):%v", round, err)
}
if set.Digest != f.writeSet.Digest {
t.Fatalf("第 %d 次接管后写集合摘要 = %s, want %s", round, set.Digest, f.writeSet.Digest)
}
}
// 证据逐项核对:写集合锚点、真实字节、作业/预备份身份都必须还在。
prebackup := f.evidenceOf(t, governance.LibraryVersionProductionPrebackup, f.backup)
for key, want := range map[string]any{
"writeset_digest": f.writeSet.Digest,
"writeset_rows": float64(2),
"job": f.jobID,
"backup_snapshot": f.backup,
"backup_bytes": float64(4096),
"index_mapping_sha256": "mapping-live",
"backup_reused": true,
} {
if got := prebackup[key]; got != want {
t.Fatalf("复用后证据 %s = %v, want %v(整份覆盖就是本缺陷)", key, got, want)
}
}
if got := f.createdAtOf(t, governance.LibraryVersionProductionPrebackup, f.backup); !got.Equal(firstCreatedAt) {
t.Fatalf("复用后预备份登记时刻 = %s, want %s(备份什么时候拍的被洗掉了)", got, firstCreatedAt)
}
// 第三次不中断:完整跑通,证明最后一次真实恢复仍按同一份写集合执行。
final := &completeTarget{}
outcome := libraryrelease.Run(ctx, f.takeoverRequest(time.Now().UTC()), final, f.takeoverBackups(t, store), f.record)
if outcome.State != "succeeded" {
t.Fatalf("最终恢复 state=%q error=%s rollback=%s", outcome.State, outcome.Error, outcome.Rollback)
}
if final.applied != len(libraryrelease.Stages()) {
t.Fatalf("四类写入没有全部执行:applied=%d", final.applied)
}
// 生效版本证据由处理器用 outcome.Backup 构造(这里不重复实现那一层),
// 所以断言的是它拿到的输入:复用路径必须带回真实字节与同一份预备份身份。
if outcome.Backup.Bytes != 4096 {
t.Fatalf("复用路径交回的备份字节 = %d, want 4096(处理器会把 0 写进生效版本证据)",
outcome.Backup.Bytes)
}
if !outcome.Backup.Reused || outcome.Backup.SnapshotID != f.backup {
t.Fatalf("复用路径交回的预备份身份不对:%+v", outcome.Backup)
}
effective := effectiveEvidence(outcome.Backup, "restore", "mapping-live")
if effective["backup_bytes"] != int64(4096) || effective["backup_snapshot"] != f.backup {
t.Fatalf("生效版本证据 = %v, want backup_bytes 4096 + backup_snapshot %s", effective, f.backup)
}
}
// 负例:复用声明与已持久化证据对不上时,必须 fail closed —— 既不许凭空登记,
// 也不许借用别的作业/别的预备份的记录。
func TestTakeoverLedgerRefusesForeignOrMissingBackupEvidence(t *testing.T) {
f := newResumeLedgerFixture(t, "negatives")
store := f.store
ctx := context.Background()
// 声称复用一份从没登记过的预备份:没有证据可合并 ⇒ 拒绝。
if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-unknown",
SnapshotID: "backup-snap-never-registered", Bytes: 1, Reused: true}); err == nil {
t.Fatal("复用未登记的预备份必须失败(否则就是把复用洗成新备份)")
}
// 同一份预备份被别的作业占用:LibraryVersionByRef 必须报身份冲突,而不是当没有。
if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-" + f.backup,
SnapshotID: f.backup, Bytes: 4096, Reused: true}); err != nil {
t.Fatalf("本方复用同一份预备份应当成功:%v", err)
}
// 字节数对不上(本次声称 2048,已登记 4096)⇒ 拒绝:宁可保持维护,也不许写错基线。
if err := f.record.BackupPrepared(ctx, libraryrelease.BackupRef{ID: "backup-" + f.backup,
SnapshotID: f.backup, Bytes: 2048, Reused: true}); err == nil {
t.Fatal("复用预备份字节数不一致时必须失败")
}
// 写集合被篡改:文件内容改一个字节,摘要就对不上 ⇒ 拒绝恢复。
path := filepath.Join(f.root, "writesets", f.writeSet.Digest+".json")
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read write set file: %v", err)
}
tampered := []byte(string(raw[:len(raw)-2]) + "X}")
// 落盘的写集合是 0444(只读台账):篡改必须先放开权限,这也顺带证明它默认不可写。
if err := os.Chmod(path, 0600); err != nil {
t.Fatalf("chmod write set: %v", err)
}
if err := os.WriteFile(path, tampered, 0600); err != nil {
t.Fatalf("tamper write set: %v", err)
}
if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup); err == nil {
t.Fatal("写集合被篡改后必须拒绝取回")
}
if err := os.WriteFile(path, raw, 0600); err != nil {
t.Fatalf("restore write set: %v", err)
}
if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, f.backup); err != nil {
t.Fatalf("恢复原文件后应能取回:%v", err)
}
// 换了预备份身份 / 换了作业:都不许借到这份记录。
if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, f.jobID, "backup-snap-other"); err == nil {
t.Fatal("不同预备份必须拒绝")
}
if _, err := loadWriteSetForJob(ctx, store, f.root, f.scope, "librestore-internal-ws-other", f.backup); err == nil {
t.Fatal("不同作业必须拒绝")
}
}