e0768f2caa
build / build (push) Successful in 1m34s
失败恢复会在写完之后按"目标租户里没人引用"清理资源行,于是历史孤儿、同租户其它 知识库暂未绑定的资源、被共享的资源都可能被删掉;而预备份里没有这些行,删了不可 恢复,随后整实例指纹才发现不一致,只能停在 rollback_failed。恢复校验依赖的整实例 证据又只存在内存里,worker 重启后只按快照 id 恢复,等于静默跳过比对。 改法:写第一笔之前把"本次明确写集合 + 写前存在状态 + 预备份时刻整实例内容证据" 算出来并按内容寻址落盘(文件名与治理库里的引用都是 sha256),摘要登记在预备份 版本证据里;恢复只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复内容里"的精确 ID,并对 资源额外要求仍无绑定/正文引用;缺记录、记录被改、绑定对不上都直接失败、维护保持。 resources 的 upsert 补租户限定,跨范围同 ID 在写入之前就被预检拒绝。 整实例指纹从"身份集合"改成"逐行内容摘要",别的知识库同 ID 行被覆盖写坏也会暴露。 本地验证(已跑): - make verify 15 项全 success(identity 因本地档案 skipped) - go test ./internal/releaseadapter/v071/ -run Isolated(真实 PG + 真实 ES + 真实 MinIO 对象存储,写失败按定位符注入):目标非空/同租户其它知识库/共享资源/ 历史孤儿/跨租户数据在失败恢复后逐行字节不变、本次残留 0;缺写集合拒绝恢复; 跨范围 ID 冲突在写入前拒绝;wiki 阶段半写回滚干净 - SIGKILL 子进程后按盘上写集合接管恢复,目标原有数据逐行不变 - go test ./internal/libraryrelease/:写集合先于第一笔、规划失败/登记失败都不写目标、 接管不重新规划
187 lines
7.0 KiB
Go
187 lines
7.0 KiB
Go
package v071
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/libraryrelease"
|
|
)
|
|
|
|
// 真杀进程的接管验证:子进程在 resources 阶段被 SIGKILL(模拟 worker 被杀),
|
|
// 已经提交的 database 阶段与"写集合 + 预备份"记录留在盘上;父进程作为"重启后的
|
|
// worker"只按持久记录恢复。
|
|
//
|
|
// 说明边界:杀的是**发布路径的进程**(本文件的 helper),不是治理栈里的 worker 容器;
|
|
// 队列/UI 侧的接管由父负责页面验收。这里证明的是"进程没了之后,恢复只用盘上的
|
|
// 写集合与整实例证据,并且删不到目标原有数据"。
|
|
|
|
const (
|
|
killChildEnv = "KG_WRITESET_KILL_CHILD"
|
|
killStateEnv = "KG_WRITESET_KILL_STATE"
|
|
killJournalEnv = "KG_WRITESET_KILL_JOURNAL"
|
|
)
|
|
|
|
// killHelperState 是子进程写下的"治理库记录"替身:真实系统里这两项分别落在
|
|
// library_jobs.snapshot_id 与 library_versions.evidence.writeset_digest。
|
|
type killHelperState struct {
|
|
JobID string `json:"job_id"`
|
|
BackupSnapshotID string `json:"backup_snapshot_id"`
|
|
TargetSnapshotID string `json:"target_snapshot_id"`
|
|
WriteSetDigest string `json:"writeset_digest"`
|
|
}
|
|
|
|
func TestWriteSetKillHelper(t *testing.T) {
|
|
if os.Getenv(killChildEnv) != "1" {
|
|
t.Skip("helper process only")
|
|
}
|
|
// 子进程复用父进程的现场:不重建 schema/索引,写集合与预备份仍落在同一个归档根。
|
|
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{archiveRoot: os.Getenv(killJournalEnv + "_ARCHIVE"),
|
|
index: os.Getenv(killJournalEnv + "_INDEX")})
|
|
state := os.Getenv(killStateEnv)
|
|
journal := os.Getenv(killJournalEnv)
|
|
snapshotID := os.Getenv("KG_WRITESET_KILL_SNAPSHOT")
|
|
target := fx.target(t)
|
|
backups := fx.backups(t, target)
|
|
current, err := target.Current(fx.ctx)
|
|
if err != nil {
|
|
t.Fatalf("current: %v", err)
|
|
}
|
|
backup, err := backups.Prepare(fx.ctx, snapshotID, current)
|
|
if err != nil {
|
|
t.Fatalf("prepare backup: %v", err)
|
|
}
|
|
set, err := target.PlanWriteSet(fx.ctx, libraryrelease.WriteSetPlan{JobID: "job-kill", Target: "internal",
|
|
BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: snapshotID, Universe: backup.Universe})
|
|
if err != nil {
|
|
t.Fatalf("plan write set: %v", err)
|
|
}
|
|
record, err := json.Marshal(killHelperState{JobID: set.JobID, BackupSnapshotID: backup.SnapshotID,
|
|
TargetSnapshotID: snapshotID, WriteSetDigest: set.Digest})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(state, record, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(journal, []byte(fx.archiveRoot), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := target.Apply(fx.ctx, libraryrelease.StageDatabase, snapshotID); err != nil {
|
|
t.Fatalf("apply database: %v", err)
|
|
}
|
|
// resources 阶段:第一笔上传就把自己杀掉(真 SIGKILL,没有 defer、没有收尾)。
|
|
dying := &killOnPutStore{inner: fx.objects.s3}
|
|
target.config.Objects = dying
|
|
if err := target.Apply(fx.ctx, libraryrelease.StageResources, snapshotID); err != nil {
|
|
t.Fatalf("apply resources: %v", err)
|
|
}
|
|
t.Fatal("helper should have been killed during the resources stage")
|
|
}
|
|
|
|
// killOnPutStore 在第一次上传时直接 SIGKILL 自己:模拟进程被外部杀掉。
|
|
type killOnPutStore struct {
|
|
inner *S3ObjectStore
|
|
}
|
|
|
|
func (s *killOnPutStore) Put(_ context.Context, _, _ string, _ io.Reader, _ int64) error {
|
|
if err := syscall.Kill(os.Getpid(), syscall.SIGKILL); err != nil {
|
|
return err
|
|
}
|
|
select {}
|
|
}
|
|
|
|
func (s *killOnPutStore) Stat(ctx context.Context, provider, path string) (int64, string, error) {
|
|
return s.inner.Stat(ctx, provider, path)
|
|
}
|
|
|
|
func (s *killOnPutStore) Copy(ctx context.Context, provider, source, target string) error {
|
|
return s.inner.Copy(ctx, provider, source, target)
|
|
}
|
|
|
|
func TestLibraryKillWorkerThenResumeRecoversWriteSetIsolated(t *testing.T) {
|
|
fx := openWriteSetFixture(t)
|
|
seedTargetData(t, fx)
|
|
before := universeDump(t, fx.pool)
|
|
snapshotID := sealSourceSnapshot(t, fx, nil)
|
|
|
|
statePath := t.TempDir() + "/kill.json"
|
|
journalPath := t.TempDir() + "/journal.txt"
|
|
command := exec.Command(os.Args[0], "-test.run=TestWriteSetKillHelper", "-test.v")
|
|
command.Env = append(os.Environ(), killChildEnv+"=1", killStateEnv+"="+statePath,
|
|
killJournalEnv+"="+journalPath, killJournalEnv+"_ARCHIVE="+fx.archiveRoot,
|
|
killJournalEnv+"_INDEX="+fx.index, "KG_WRITESET_KILL_SNAPSHOT="+snapshotID)
|
|
output, err := command.CombinedOutput()
|
|
if err == nil {
|
|
t.Fatalf("helper must be killed, it exited cleanly: %s", output)
|
|
}
|
|
exitErr, ok := err.(*exec.ExitError)
|
|
if !ok {
|
|
t.Fatalf("helper failed to run: %v\n%s", err, output)
|
|
}
|
|
status, ok := exitErr.Sys().(syscall.WaitStatus)
|
|
if !ok || !status.Signaled() || status.Signal() != syscall.SIGKILL {
|
|
t.Fatalf("helper must die from SIGKILL, got %v\n%s", err, output)
|
|
}
|
|
t.Logf("helper killed by %s (half-written target)", status.Signal())
|
|
|
|
raw, err := os.ReadFile(statePath)
|
|
if err != nil {
|
|
t.Fatalf("child did not persist the job record: %v", err)
|
|
}
|
|
var record killHelperState
|
|
if err := json.Unmarshal(raw, &record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if record.WriteSetDigest == "" || record.BackupSnapshotID == "" {
|
|
t.Fatalf("child must persist writeset digest + backup snapshot: %+v", record)
|
|
}
|
|
journalRoot, err := os.ReadFile(journalPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.TrimSpace(string(journalRoot)) != fx.archiveRoot {
|
|
t.Fatalf("journal root mismatch: %q", journalRoot)
|
|
}
|
|
|
|
// 重启后的 worker:新连接、新目标对象,只拿盘上的记录。
|
|
resumedTarget := fx.target(t)
|
|
ledger, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest)
|
|
if err != nil {
|
|
t.Fatalf("重启后取回写集合失败:%v", err)
|
|
}
|
|
if ledger.JobID != record.JobID || ledger.BackupSnapshotID != record.BackupSnapshotID ||
|
|
ledger.TargetSnapshotID != record.TargetSnapshotID {
|
|
t.Fatalf("写集合与作业记录不一致:%+v vs %+v", ledger, record)
|
|
}
|
|
if err := resumedTarget.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + record.BackupSnapshotID,
|
|
SnapshotID: record.BackupSnapshotID, WriteSet: ledger}); err != nil {
|
|
t.Fatalf("接管恢复失败:%v", err)
|
|
}
|
|
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
|
t.Fatalf("接管恢复后目标数据被改动:%s", diff)
|
|
}
|
|
if err := resumedTarget.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: record.BackupSnapshotID}); err != nil {
|
|
t.Fatalf("接管恢复后必须与预备份一致:%v", err)
|
|
}
|
|
t.Logf("resumed after SIGKILL: job=%s backup=%s ledger=%s", record.JobID, record.BackupSnapshotID, record.WriteSetDigest)
|
|
|
|
// 负例:记录被移除 → 恢复必须失败(维护保持),而不是"看不见就当没写过"。
|
|
if err := os.Chmod(writeSetPath(fx.archiveRoot, record.WriteSetDigest), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Remove(writeSetPath(fx.archiveRoot, record.WriteSetDigest)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest); err == nil {
|
|
t.Fatal("写集合被移除后必须取不回来")
|
|
}
|
|
_ = time.Now()
|
|
}
|