package v071 import ( "context" "encoding/json" "io" "os" "os/exec" "strings" "syscall" "testing" "time" "knowledge-governance/internal/libraryrelease" ) // 真杀进程的接管验证:子进程在 resources 阶段被 SIGKILL(模拟 worker 被杀), // 已经提交的 database 阶段与"写集合 + 预备份"记录留在盘上;父进程作为"重启后的 // worker"只按持久记录恢复。 // // 说明边界:杀的是**发布路径的进程**(本文件的 helper),不是治理栈里的 worker 容器; // 队列/UI 侧的接管由父负责页面验收。这里证明的是"进程没了之后,恢复只用盘上的 // 写集合与整实例证据,并且删不到目标原有数据"。 const ( killChildEnv = "KG_WRITESET_KILL_CHILD" killStateEnv = "KG_WRITESET_KILL_STATE" killJournalEnv = "KG_WRITESET_KILL_JOURNAL" ) // killHelperState 是子进程写下的"治理库记录"替身:真实系统里这两项分别落在 // library_jobs.snapshot_id 与 library_versions.evidence.writeset_digest。 type killHelperState struct { JobID string `json:"job_id"` BackupSnapshotID string `json:"backup_snapshot_id"` TargetSnapshotID string `json:"target_snapshot_id"` WriteSetDigest string `json:"writeset_digest"` } func TestWriteSetKillHelper(t *testing.T) { if os.Getenv(killChildEnv) != "1" { t.Skip("helper process only") } // 子进程复用父进程的现场:不重建 schema/索引,写集合与预备份仍落在同一个归档根。 fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{archiveRoot: os.Getenv(killJournalEnv + "_ARCHIVE"), index: os.Getenv(killJournalEnv + "_INDEX")}) state := os.Getenv(killStateEnv) journal := os.Getenv(killJournalEnv) snapshotID := os.Getenv("KG_WRITESET_KILL_SNAPSHOT") target := fx.target(t) backups := fx.backups(t, target) current, err := target.Current(fx.ctx) if err != nil { t.Fatalf("current: %v", err) } backup, err := backups.Prepare(fx.ctx, snapshotID, current) if err != nil { t.Fatalf("prepare backup: %v", err) } set, err := target.PlanWriteSet(fx.ctx, libraryrelease.WriteSetPlan{JobID: "job-kill", Target: "internal", BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: snapshotID, Universe: backup.Universe}) if err != nil { t.Fatalf("plan write set: %v", err) } record, err := json.Marshal(killHelperState{JobID: set.JobID, BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: snapshotID, WriteSetDigest: set.Digest}) if err != nil { t.Fatal(err) } if err := os.WriteFile(state, record, 0600); err != nil { t.Fatal(err) } if err := os.WriteFile(journal, []byte(fx.archiveRoot), 0600); err != nil { t.Fatal(err) } if err := target.Apply(fx.ctx, libraryrelease.StageDatabase, snapshotID); err != nil { t.Fatalf("apply database: %v", err) } // resources 阶段:第一笔上传就把自己杀掉(真 SIGKILL,没有 defer、没有收尾)。 dying := &killOnPutStore{inner: fx.objects.s3} target.config.Objects = dying if err := target.Apply(fx.ctx, libraryrelease.StageResources, snapshotID); err != nil { t.Fatalf("apply resources: %v", err) } t.Fatal("helper should have been killed during the resources stage") } // killOnPutStore 在第一次上传时直接 SIGKILL 自己:模拟进程被外部杀掉。 type killOnPutStore struct { inner *S3ObjectStore } func (s *killOnPutStore) Put(_ context.Context, _, _ string, _ io.Reader, _ int64) error { if err := syscall.Kill(os.Getpid(), syscall.SIGKILL); err != nil { return err } select {} } func (s *killOnPutStore) Stat(ctx context.Context, provider, path string) (int64, string, error) { return s.inner.Stat(ctx, provider, path) } func (s *killOnPutStore) Copy(ctx context.Context, provider, source, target string) error { return s.inner.Copy(ctx, provider, source, target) } func TestLibraryKillWorkerThenResumeRecoversWriteSetIsolated(t *testing.T) { fx := openWriteSetFixture(t) seedTargetData(t, fx) before := universeDump(t, fx.pool) snapshotID := sealSourceSnapshot(t, fx, nil) statePath := t.TempDir() + "/kill.json" journalPath := t.TempDir() + "/journal.txt" command := exec.Command(os.Args[0], "-test.run=TestWriteSetKillHelper", "-test.v") command.Env = append(os.Environ(), killChildEnv+"=1", killStateEnv+"="+statePath, killJournalEnv+"="+journalPath, killJournalEnv+"_ARCHIVE="+fx.archiveRoot, killJournalEnv+"_INDEX="+fx.index, "KG_WRITESET_KILL_SNAPSHOT="+snapshotID) output, err := command.CombinedOutput() if err == nil { t.Fatalf("helper must be killed, it exited cleanly: %s", output) } exitErr, ok := err.(*exec.ExitError) if !ok { t.Fatalf("helper failed to run: %v\n%s", err, output) } status, ok := exitErr.Sys().(syscall.WaitStatus) if !ok || !status.Signaled() || status.Signal() != syscall.SIGKILL { t.Fatalf("helper must die from SIGKILL, got %v\n%s", err, output) } t.Logf("helper killed by %s (half-written target)", status.Signal()) raw, err := os.ReadFile(statePath) if err != nil { t.Fatalf("child did not persist the job record: %v", err) } var record killHelperState if err := json.Unmarshal(raw, &record); err != nil { t.Fatal(err) } if record.WriteSetDigest == "" || record.BackupSnapshotID == "" { t.Fatalf("child must persist writeset digest + backup snapshot: %+v", record) } journalRoot, err := os.ReadFile(journalPath) if err != nil { t.Fatal(err) } if strings.TrimSpace(string(journalRoot)) != fx.archiveRoot { t.Fatalf("journal root mismatch: %q", journalRoot) } // 重启后的 worker:新连接、新目标对象,只拿盘上的记录。 resumedTarget := fx.target(t) ledger, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest) if err != nil { t.Fatalf("重启后取回写集合失败:%v", err) } if ledger.JobID != record.JobID || ledger.BackupSnapshotID != record.BackupSnapshotID || ledger.TargetSnapshotID != record.TargetSnapshotID { t.Fatalf("写集合与作业记录不一致:%+v vs %+v", ledger, record) } if err := resumedTarget.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + record.BackupSnapshotID, SnapshotID: record.BackupSnapshotID, WriteSet: ledger}); err != nil { t.Fatalf("接管恢复失败:%v", err) } if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { t.Fatalf("接管恢复后目标数据被改动:%s", diff) } if err := resumedTarget.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: record.BackupSnapshotID}); err != nil { t.Fatalf("接管恢复后必须与预备份一致:%v", err) } t.Logf("resumed after SIGKILL: job=%s backup=%s ledger=%s", record.JobID, record.BackupSnapshotID, record.WriteSetDigest) // 负例:记录被移除 → 恢复必须失败(维护保持),而不是"看不见就当没写过"。 if err := os.Chmod(writeSetPath(fx.archiveRoot, record.WriteSetDigest), 0600); err != nil { t.Fatal(err) } if err := os.Remove(writeSetPath(fx.archiveRoot, record.WriteSetDigest)); err != nil { t.Fatal(err) } if _, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest); err == nil { t.Fatal("写集合被移除后必须取不回来") } _ = time.Now() }