diff --git a/cmd/librarydrill/main.go b/cmd/librarydrill/main.go index da6e78c..be21d1d 100644 --- a/cmd/librarydrill/main.go +++ b/cmd/librarydrill/main.go @@ -28,6 +28,7 @@ import ( "knowledge-governance/internal/libraryfreeze" "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarystack" + v071 "knowledge-governance/internal/releaseadapter/v071" ) type drillVerification struct { @@ -220,7 +221,21 @@ WHERE id = (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$ log.Printf("drill: drift detected as expected (%s)", drift.Error) // 4) 按预备份恢复。 - if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID}); err != nil { + // 恢复必须带着写集合记录。演练只改了一行既有内容(canary),没有新建行, + // 所以写集合为空;整实例证据用预备份时刻的那一份。 + identityValue := target.Identity() + ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope, + TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID, + BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID, + CreatedAt: time.Now().UTC(), Universe: backup.Universe} + if err := ledger.Seal(); err != nil { + return "", errors.New("演练写集合生成失败:" + err.Error()) + } + if _, err := v071.PersistWriteSet(config.LibraryArchiveRoot, &ledger); err != nil { + return "", errors.New("演练写集合落盘失败:" + err.Error()) + } + if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID, + WriteSet: &ledger}); err != nil { return "", errors.New("按预备份恢复失败:" + err.Error()) } // 5) 恢复后逐类复核:必须与预备份完全一致。 diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index 87ba6d8..64a0d90 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -291,12 +291,24 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a var backupSource libraryrelease.BackupStore = backups if resumed { - backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID, - SnapshotID: previous.SnapshotID}} + // 接管必须带着原来那份写集合:没有它,恢复只能按目标现状推断残渣, + // 而那时目标已经被写过(真实缺陷:历史孤儿资源被当成残渣删掉)。 + set, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope, + payload.LibraryJobID, previous.SnapshotID) + if ledgerErr != nil { + // 读不到就一笔都不许写:目标可能半写,恢复口径未知。 + message := "接管时读不回写集合记录,拒绝继续写目标:" + ledgerErr.Error() + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC()) + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC()) + recorder.Stage(ctx, "prebackup", "failed", message) + return errors.New(message) + } + backupSource = ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID, + SnapshotID: previous.SnapshotID}, write: set} } - request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID, - ParamsDigest: digest, Now: now, + request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope, + SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now, Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID, ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}} _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now) @@ -506,12 +518,6 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Stor } // fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。 -type fixedBackupStore struct{ ref libraryrelease.BackupRef } - -func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) { - return s.ref, nil -} - // BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误: // 上游据此在写目标前中止,保证"没有可恢复基线就不写"。 func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error { diff --git a/cmd/worker/library_recovery.go b/cmd/worker/library_recovery.go index 439d641..14832cb 100644 --- a/cmd/worker/library_recovery.go +++ b/cmd/worker/library_recovery.go @@ -64,6 +64,16 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份") } prebackup := windowJob.SnapshotID + // 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣, + // 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。 + ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope, + payload.WindowJobID, prebackup) + if ledgerErr != nil { + note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error() + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) + return errors.New(note) + } knownMapping := knownIndexMapping(ctx, store, payload.Scope) target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping) if wireErr != nil { @@ -78,7 +88,7 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config defer control.Close() defer coordinator.Close() - restorer := recoveryRestorer{target: target, expected: prebackup} + restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger} verifier := recoveryVerifier{target: target, expected: prebackup} result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier) now := time.Now().UTC() @@ -106,13 +116,21 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config type recoveryRestorer struct { target *v071.ProductionLibraryTarget expected string + writeSet *libraryrelease.WriteSet } func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error { if backupID != r.expected { return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected) } - return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID}) + if r.writeSet == nil { + return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣") + } + if r.writeSet.BackupSnapshotID != backupID { + return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID) + } + return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID, + WriteSet: r.writeSet}) } // recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。 diff --git a/cmd/worker/library_writeset.go b/cmd/worker/library_writeset.go new file mode 100644 index 0000000..77eaa06 --- /dev/null +++ b/cmd/worker/library_writeset.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/libraryrelease" + postgresstore "knowledge-governance/internal/platform/postgres" + v071 "knowledge-governance/internal/releaseadapter/v071" +) + +// 写集合台账在治理侧的锚点:摘要登记在"这次作业的预备份版本"证据里。 +// 为什么不用新表:版本记录本来就绑定 scope + snapshot + 作业,且 id 由这三者 +// 决定;把摘要挂上去,重启后按同一身份就能取回,不需要改 schema。 + +// writeSetDigestOf 读某次作业/某份预备份登记的写集合摘要。 +// 读不到就返回错误:缺失必须让调用方失败,不能降级成"没有记录就当没写过"。 +func writeSetDigestOf(ctx context.Context, store *postgresstore.Store, scope, jobID, snapshotID string) (string, error) { + versions, err := store.LibraryVersions(ctx, scope, 200) + if err != nil { + return "", fmt.Errorf("读不到版本记录:%w", err) + } + for _, version := range versions { + if version.Kind != governance.LibraryVersionProductionPrebackup || version.SnapshotID != snapshotID { + continue + } + if version.SourceJobID != jobID { + continue + } + raw, _ := version.Evidence["writeset_digest"].(string) + if raw == "" { + return "", fmt.Errorf("预备份 %s 没有登记写集合摘要(作业 %s)", snapshotID, jobID) + } + return raw, nil + } + return "", fmt.Errorf("找不到作业 %s 的预备份版本记录(快照 %s)", jobID, snapshotID) +} + +// loadWriteSetForJob 按"作业 + 目标 + 预备份"取回写集合并逐项核对绑定。 +// 任何一步不符都返回错误:恢复宁可保持维护,也不许用一份对不上的记录去删目标数据。 +func loadWriteSetForJob(ctx context.Context, store *postgresstore.Store, archiveRoot, scope, jobID, snapshotID string) (*libraryrelease.WriteSet, error) { + digest, err := writeSetDigestOf(ctx, store, scope, jobID, snapshotID) + if err != nil { + return nil, err + } + set, err := v071.LoadWriteSet(archiveRoot, digest) + if err != nil { + return nil, err + } + if set.JobID != jobID { + return nil, fmt.Errorf("写集合记录属于作业 %s,不是 %s", set.JobID, jobID) + } + if set.Target != scope { + return nil, fmt.Errorf("写集合记录属于目标 %s,不是 %s", set.Target, scope) + } + if set.BackupSnapshotID != snapshotID { + return nil, fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", set.BackupSnapshotID, snapshotID) + } + return set, nil +} + +// WriteSetPlanned 把写集合摘要登记进"这次作业的预备份版本"证据。 +// 它在第一笔写入之前被调用;失败即中止(调用方据此不写目标)。 +func (r libraryStageRecorder) WriteSetPlanned(ctx context.Context, set libraryrelease.WriteSet) error { + versions, err := r.store.LibraryVersions(ctx, r.scope, 200) + if err != nil { + return fmt.Errorf("读不到现有版本证据:%w", err) + } + var existing map[string]any + for _, version := range versions { + if version.Kind == governance.LibraryVersionProductionPrebackup && version.SnapshotID == set.BackupSnapshotID { + existing = version.Evidence + break + } + } + evidence := map[string]any{} + for key, value := range existing { + evidence[key] = value + } + // 保留预备份原有证据(备份字节、索引映射等),只追加写集合锚点。 + evidence["backup_snapshot"] = set.BackupSnapshotID + evidence["job"] = set.JobID + if r.mappingDigest != "" { + evidence["index_mapping_sha256"] = r.mappingDigest + } + evidence["writeset_digest"] = set.Digest + evidence["writeset_universe_digest"] = set.UniverseDigest() + evidence["writeset_target_snapshot"] = set.TargetSnapshotID + evidence["writeset_rows"] = len(set.Rows) + return r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{ + ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, set.BackupSnapshotID), + Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: set.BackupSnapshotID, + SourceJobID: r.jobID, CreatedBy: "worker", Evidence: evidence, CreatedAt: time.Now().UTC(), + }) +} + +// ledgerBackupStore 是"接管/恢复"用的预备份来源:只认已经持久化的那一份, +// 并把它绑定的写集合一起带出来。 +// +// 为什么不能像以前那样只返回 snapshot id:那一刻目标已经被写过,恢复若拿不到 +// 写集合就只能按目标现状推断残渣——那正是会误删历史资源的路径。所以这里取不到 +// 记录就直接失败,而不是返回一个"没有 Universe 的引用"。 +type ledgerBackupStore struct { + ref libraryrelease.BackupRef + write *libraryrelease.WriteSet +} + +func (s ledgerBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) { + if s.write == nil { + return libraryrelease.BackupRef{}, errors.New("恢复缺少持久化的写集合记录:拒绝在没有可追溯写集合的情况下继续") + } + ref := s.ref + ref.WriteSet = s.write + return ref, nil +} + +// sortedWriteSetSummary 只用于日志/证据:把本次写集合按表汇总成稳定的短摘要。 +func sortedWriteSetSummary(set libraryrelease.WriteSet) string { + counts := map[string]int{} + for _, row := range set.Rows { + counts[row.Table]++ + } + tables := make([]string, 0, len(counts)) + for table := range counts { + tables = append(tables, table) + } + sort.Strings(tables) + summary := "" + for _, table := range tables { + if summary != "" { + summary += "," + } + summary += fmt.Sprintf("%s=%d", table, counts[table]) + } + return summary +} diff --git a/internal/libraryrelease/release.go b/internal/libraryrelease/release.go index 3322bf2..0de15be 100644 --- a/internal/libraryrelease/release.go +++ b/internal/libraryrelease/release.go @@ -36,6 +36,9 @@ type Grant struct { // Request 是一次待执行的生产同步。 type Request struct { + // JobID 是这次发布的作业身份:写集合记录与预备份引用都绑定它, + // worker 重启后凭它找回"这次写了什么"。 + JobID string Target string SnapshotID string // ParamsDigest 覆盖会影响生产结果的全部参数(目标、快照、排除范围等)。 @@ -75,9 +78,11 @@ type BackupRef struct { SnapshotID string `json:"snapshot_id"` Bytes int64 `json:"bytes"` CreatedAt time.Time `json:"created_at"` - // Universe 是预备份时刻的整实例指纹,恢复后逐表比对。空表示这次没有采集 - // (例如 worker 被杀后用已持久化的快照接管),此时恢复只能靠内容校验兜底。 + // Universe 是预备份时刻的整实例指纹,恢复后逐表比对。 Universe map[string]TableFingerprint `json:"universe,omitempty"` + // WriteSet 是本次发布的写集合与整实例内容证据。它在写第一笔之前就落盘并 + // 登记摘要;恢复必须按它清理,缺了它恢复要失败(不许降级成"跳过比对")。 + WriteSet *WriteSet `json:"write_set,omitempty"` } type Target interface { @@ -203,6 +208,32 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor } recorder.Stage(ctx, "prebackup", "succeeded", "预备份 "+backup.ID) + // 写集合必须在第一笔写入之前落盘并登记:恢复只按"这次明确写了什么"清理, + // 不按目标现状推断残渣(那会把历史孤儿/共享资源当成本次残留删掉)。 + if backup.WriteSet == nil { + if planner, ok := target.(WriteSetPlanner); ok { + set, planErr := planner.PlanWriteSet(ctx, WriteSetPlan{JobID: request.JobID, Target: request.Target, + BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: request.SnapshotID, Universe: backup.Universe}) + if planErr != nil { + outcome.State = "failed" + outcome.Error = "写集合/冲突预检失败,未对生产执行任何写入:" + planErr.Error() + recorder.Stage(ctx, "prebackup", "failed", outcome.Error) + return outcome + } + backup.WriteSet = &set + outcome.Backup = backup + if observer, ok := recorder.(WriteSetObserver); ok { + if err := observer.WriteSetPlanned(ctx, set); err != nil { + // 硬门禁:记录没落库 = 恢复时不知道写集合 = 一笔都不许写。 + outcome.State = "failed" + outcome.Error = "写集合已落盘但引用无法持久化,未对生产执行任何写入:" + err.Error() + recorder.Stage(ctx, "prebackup", "failed", outcome.Error) + return outcome + } + } + } + } + for _, stage := range Stages() { recorder.Stage(ctx, string(stage), "running", "") if err = target.Apply(ctx, stage, request.SnapshotID); err != nil { diff --git a/internal/libraryrelease/release_test.go b/internal/libraryrelease/release_test.go index 824bcef..fa96dc8 100644 --- a/internal/libraryrelease/release_test.go +++ b/internal/libraryrelease/release_test.go @@ -250,3 +250,162 @@ func TestRunReportsRollbackFailureDistinctly(t *testing.T) { t.Fatalf("stage log = %v", recorder.entries) } } + +// writeSetPlanTarget 记录"写集合规划"与阶段写入的相对顺序:写集合必须在第一笔之前。 +type writeSetPlanTarget struct { + *fakeTarget + planErr error + plans []WriteSetPlan + events *[]string + ledger *WriteSet + restores []BackupRef +} + +func (p *writeSetPlanTarget) PlanWriteSet(_ context.Context, plan WriteSetPlan) (WriteSet, error) { + *p.events = append(*p.events, "plan") + p.plans = append(p.plans, plan) + if p.planErr != nil { + return WriteSet{}, p.planErr + } + set := WriteSet{Version: WriteSetVersion, JobID: plan.JobID, Target: plan.Target, + TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: plan.BackupSnapshotID, + TargetSnapshotID: plan.TargetSnapshotID, CreatedAt: time.Now().UTC(), Universe: plan.Universe, + Rows: []WriteSetRow{{Table: "resources", ID: "res-new"}}} + if err := set.Seal(); err != nil { + return WriteSet{}, err + } + p.ledger = &set + return set, nil +} + +func (p *writeSetPlanTarget) Apply(ctx context.Context, stage Stage, snapshot string) error { + *p.events = append(*p.events, "apply:"+string(stage)) + return p.fakeTarget.Apply(ctx, stage, snapshot) +} + +func (p *writeSetPlanTarget) Restore(ctx context.Context, backup BackupRef) error { + p.restores = append(p.restores, backup) + return p.fakeTarget.Restore(ctx, backup) +} + +type writeSetRecorder struct { + *stageLog + events *[]string + err error + sets []WriteSet +} + +func (r *writeSetRecorder) WriteSetPlanned(_ context.Context, set WriteSet) error { + *r.events = append(*r.events, "observe") + r.sets = append(r.sets, set) + return r.err +} + +func plannedRun(t *testing.T) (*writeSetPlanTarget, *writeSetRecorder, *[]string, Request, *fakeBackups) { + t.Helper() + now := time.Now() + events := []string{} + target := &writeSetPlanTarget{fakeTarget: &fakeTarget{}, events: &events} + recorder := &writeSetRecorder{stageLog: &stageLog{}, events: &events} + backups := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap", Bytes: 512, + CreatedAt: now.UTC(), Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}} + request := request(now) + return target, recorder, &events, request, backups +} + +func TestRunPlansWriteSetBeforeFirstWrite(t *testing.T) { + target, recorder, events, request, backups := plannedRun(t) + outcome := Run(context.Background(), request, target, backups, recorder) + if outcome.State != "succeeded" { + t.Fatalf("outcome = %+v", outcome) + } + if len(target.plans) != 1 { + t.Fatalf("write set must be planned exactly once, got %d", len(target.plans)) + } + if got := (*events)[0]; got != "plan" { + t.Fatalf("write set must be planned before anything touches the target, first event = %s", got) + } + if len(recorder.sets) != 1 || recorder.sets[0].Digest == "" { + t.Fatalf("planner output must be persisted before writing: %+v", recorder.sets) + } + if target.plans[0].JobID != request.JobID || target.plans[0].TargetSnapshotID != request.SnapshotID || + target.plans[0].Target != request.Target { + t.Fatalf("plan must bind job/target/snapshot: %+v", target.plans[0]) + } + if target.plans[0].BackupSnapshotID != "backup-snap" { + t.Fatalf("plan must bind the backup snapshot: %+v", target.plans[0]) + } + if len(*events) < 2 || (*events)[1] != "observe" { + t.Fatalf("write set must be persisted right after planning, events=%v", *events) + } +} + +func TestRunStopsBeforeWritingWhenWriteSetPlanFails(t *testing.T) { + target, recorder, events, request, backups := plannedRun(t) + target.planErr = errors.New("资源 ID 冲突:res-x 已属于其它知识库") + outcome := Run(context.Background(), request, target, backups, recorder) + if outcome.State != "failed" { + t.Fatalf("outcome = %+v", outcome) + } + if !strings.Contains(outcome.Error, "冲突") { + t.Fatalf("error must carry the conflict reason: %s", outcome.Error) + } + if len(target.applied) != 0 { + t.Fatalf("no stage may run after a failed preflight: %v", target.applied) + } + if len(recorder.sets) != 0 { + t.Fatal("a failed plan must not be persisted as the write set") + } + for _, event := range *events { + if strings.HasPrefix(event, "apply:") { + t.Fatalf("target was written despite a failed preflight: %v", *events) + } + } +} + +func TestRunStopsBeforeWritingWhenWriteSetCannotBePersisted(t *testing.T) { + target, recorder, events, request, backups := plannedRun(t) + recorder.err = errors.New("治理库写不进去") + outcome := Run(context.Background(), request, target, backups, recorder) + if outcome.State != "failed" { + t.Fatalf("outcome = %+v", outcome) + } + if !strings.Contains(outcome.Error, "写集合") { + t.Fatalf("error must name the write-set record: %s", outcome.Error) + } + if len(target.applied) != 0 { + t.Fatalf("no stage may run when the write set cannot be persisted: %v", target.applied) + } + for _, event := range *events { + if strings.HasPrefix(event, "apply:") { + t.Fatalf("target was written without a persisted write set: %v", *events) + } + } +} + +// 接管时必须复用持久化的写集合,而不是按当前(已半写的)目标重新规划。 +func TestRunResumeUsesPersistedWriteSet(t *testing.T) { + target, recorder, events, request, backups := plannedRun(t) + resumed := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap", + WriteSet: &WriteSet{Version: WriteSetVersion, JobID: request.JobID, Target: request.Target, + TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: "backup-snap", + TargetSnapshotID: request.SnapshotID, CreatedAt: time.Now().UTC(), + Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}}} + backups = resumed + target.fakeTarget.applyErr = map[Stage]error{StageResources: errors.New("上传失败")} + outcome := Run(context.Background(), request, target, backups, recorder) + if outcome.State != "rolled_back" && outcome.State != "rollback_failed" { + t.Fatalf("outcome = %+v", outcome) + } + if len(target.plans) != 0 { + t.Fatalf("resume must not re-plan the write set: %+v", target.plans) + } + if len(target.restores) != 1 || target.restores[0].WriteSet == nil { + t.Fatalf("restore must carry the persisted write set: %+v", target.restores) + } + for _, event := range *events { + if event == "plan" { + t.Fatalf("resume re-planned against a half-written target: %v", *events) + } + } +} diff --git a/internal/libraryrelease/writeset.go b/internal/libraryrelease/writeset.go new file mode 100644 index 0000000..89c1ba1 --- /dev/null +++ b/internal/libraryrelease/writeset.go @@ -0,0 +1,200 @@ +package libraryrelease + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "time" +) + +// WriteSetVersion 是写集合台账的格式版本。读到的版本不认识就拒绝恢复: +// 宁可保持维护,也不要用一份"格式变了但看起来还能读"的记录去删目标数据。 +const WriteSetVersion = 1 + +// WriteSetRow 是台账里的一行:本次发布明确要写的精确身份 + 写前状态。 +// +// 为什么需要"写前状态":恢复不能在目标里按"没人引用"推断残渣。历史孤儿资源、 +// 同租户另一个知识库暂未绑定的资源、被多个知识库共享的资源,都满足"没人引用", +// 但它们不是这次作业写的,删掉就是数据丢失(快照里没有这些行,恢复不回来)。 +// 只有写前不存在、且由本次写进去的行才允许清理。 +type WriteSetRow struct { + Table string `json:"table"` + ID string `json:"id"` + // Existing 表示写前目标里已经有这一行(属于目标自己的数据,恢复一律不许删)。 + Existing bool `json:"existing"` + // Owner 是写前该行的归属("/"),不存在时为空。 + Owner string `json:"owner,omitempty"` +} + +// WriteSetPlan 是调用方在写第一笔之前交给目标的信息:绑定作业、目标、预备份与快照, +// 以及预备份时刻的整实例证据。 +type WriteSetPlan struct { + JobID string + Target string + BackupSnapshotID string + TargetSnapshotID string + Universe map[string]TableFingerprint +} + +// WriteSet 是"这一次发布的写集合 + 写前整实例内容证据",内容寻址、可校验、 +// 与 job/target/backup 绑定。恢复只读它,不读目标现状去推断。 +type WriteSet struct { + Version int `json:"version"` + JobID string `json:"job_id"` + Target string `json:"target"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + BackupSnapshotID string `json:"backup_snapshot_id"` + TargetSnapshotID string `json:"target_snapshot_id"` + CreatedAt time.Time `json:"created_at"` + Rows []WriteSetRow `json:"rows"` + Universe map[string]TableFingerprint `json:"universe"` + // Digest 是对除本字段外全部内容的 sha256:落盘文件名与治理库里的引用都用它, + // 记录被改一个字节就对不上。 + Digest string `json:"digest"` +} + +// Seal 把行序规范化为可复现的顺序并写入摘要。调用方在落盘与登记之前必须先 Seal。 +// 排序在这里落地(而不是只在算摘要时排),否则落盘字节与摘要覆盖的内容不一致。 +func (w *WriteSet) Seal() error { + sort.Slice(w.Rows, func(i, j int) bool { + if w.Rows[i].Table != w.Rows[j].Table { + return w.Rows[i].Table < w.Rows[j].Table + } + return w.Rows[i].ID < w.Rows[j].ID + }) + digest, err := w.computeDigest() + if err != nil { + return err + } + w.Digest = digest + return nil +} + +// CanonicalBytes 是不含摘要字段的规范字节:摘要覆盖的就是它,落盘文件另外 +// 带上摘要本身(便于自描述),读回时用同一份规范字节重新推导。 +func (w WriteSet) CanonicalBytes() ([]byte, error) { + copied := w + copied.Digest = "" + rows := make([]WriteSetRow, len(w.Rows)) + copy(rows, w.Rows) + // 行序不影响语义但影响摘要:两边都按 (table,id) 排序才能算出同一个值。 + sort.Slice(rows, func(i, j int) bool { + if rows[i].Table != rows[j].Table { + return rows[i].Table < rows[j].Table + } + return rows[i].ID < rows[j].ID + }) + copied.Rows = rows + raw, err := json.Marshal(copied) + if err != nil { + return nil, fmt.Errorf("编码写集合失败: %w", err) + } + return raw, nil +} + +func (w *WriteSet) computeDigest() (string, error) { + raw, err := w.CanonicalBytes() + if err != nil { + return "", err + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]), nil +} + +// Verify 校验摘要与必备身份。缺任一身份都必须失败:没有绑定就没有"这份记录属于 +// 这个作业/目标/预备份"的证明,拿它去删目标数据无法追责。 +func (w *WriteSet) Verify() error { + if w.Version != WriteSetVersion { + return fmt.Errorf("写集合记录版本 %d 不受支持(当前 %d)", w.Version, WriteSetVersion) + } + if w.JobID == "" || w.BackupSnapshotID == "" || w.TargetSnapshotID == "" { + return errors.New("写集合记录缺少作业/预备份/快照绑定") + } + if w.TenantID == 0 || w.KnowledgeBaseID == "" { + return errors.New("写集合记录缺少目标范围身份") + } + if len(w.Universe) == 0 { + return errors.New("写集合记录缺少整实例内容证据") + } + if w.Digest == "" { + return errors.New("写集合记录没有摘要") + } + digest, err := w.computeDigest() + if err != nil { + return err + } + if digest != w.Digest { + return errors.New("写集合记录摘要不符(内容被改动或截断)") + } + return nil +} + +// CreatedIDs 返回"本次新建"的精确身份(写前不存在的行)。恢复只允许删这些。 +func (w WriteSet) CreatedIDs(table string) []string { + ids := make([]string, 0, 8) + for _, row := range w.Rows { + if row.Table == table && !row.Existing { + ids = append(ids, row.ID) + } + } + sort.Strings(ids) + return ids +} + +// ExistingIDs 返回写前已存在的身份(用于恢复后复核"目标自己的行都还在")。 +func (w WriteSet) ExistingIDs(table string) []string { + ids := make([]string, 0, 8) + for _, row := range w.Rows { + if row.Table == table && row.Existing { + ids = append(ids, row.ID) + } + } + sort.Strings(ids) + return ids +} + +// UniverseDigest 是整实例证据的摘要,随台账一起登记,便于独立核对。 +func (w WriteSet) UniverseDigest() string { + if len(w.Universe) == 0 { + return "" + } + tables := make([]string, 0, len(w.Universe)) + for table := range w.Universe { + tables = append(tables, table) + } + sort.Strings(tables) + type pair struct { + Count int64 `json:"count"` + Digest string `json:"digest"` + } + ordered := make([]pair, 0, len(tables)) + for _, table := range tables { + ordered = append(ordered, pair{Count: w.Universe[table].Count, Digest: w.Universe[table].Digest}) + } + raw, err := json.Marshal(struct { + Tables []string `json:"tables"` + Values []pair `json:"values"` + }{Tables: tables, Values: ordered}) + if err != nil { + return "" + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} + +// WriteSetPlanner 由能访问目标与归档的实现提供:它在写第一笔之前算出写集合、 +// 拒绝跨范围 ID 冲突,并把记录落盘。失败即"一笔都不许写"。 +type WriteSetPlanner interface { + PlanWriteSet(ctx context.Context, plan WriteSetPlan) (WriteSet, error) +} + +// WriteSetObserver 让调用方在写第一笔之前把写集合摘要持久化到治理库。 +// 失败同样意味着"一笔都不许写":没有可追溯的写集合,恢复就只能靠推断。 +type WriteSetObserver interface { + WriteSetPlanned(ctx context.Context, set WriteSet) error +} diff --git a/internal/libraryrelease/writeset_test.go b/internal/libraryrelease/writeset_test.go new file mode 100644 index 0000000..1a01d28 --- /dev/null +++ b/internal/libraryrelease/writeset_test.go @@ -0,0 +1,104 @@ +package libraryrelease + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func sampleWriteSet() WriteSet { + return WriteSet{Version: WriteSetVersion, JobID: "job-1", Target: "internal", TenantID: 10001, + KnowledgeBaseID: "kb", BackupSnapshotID: "backup-1", TargetSnapshotID: "snap-1", + CreatedAt: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC), + Rows: []WriteSetRow{ + {Table: "resources", ID: "res-new"}, + {Table: "knowledges", ID: "doc-existing", Existing: true, Owner: "10001/kb"}, + {Table: "resources", ID: "res-old", Existing: true, Owner: "10001/"}, + }, + Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "abc"}}} +} + +func TestWriteSetSealVerifyRoundTrip(t *testing.T) { + set := sampleWriteSet() + if err := set.Seal(); err != nil { + t.Fatal(err) + } + if set.Digest == "" { + t.Fatal("seal must produce a digest") + } + if err := set.Verify(); err != nil { + t.Fatalf("sealed record must verify: %v", err) + } + // 行序不影响摘要:换序后重新 Sealed 必须得到同一个值(落盘/读回才可比)。 + shuffled := sampleWriteSet() + shuffled.Rows[0], shuffled.Rows[2] = shuffled.Rows[2], shuffled.Rows[0] + if err := shuffled.Seal(); err != nil { + t.Fatal(err) + } + if shuffled.Digest != set.Digest { + t.Fatalf("digest must not depend on row order: %s vs %s", shuffled.Digest, set.Digest) + } + // 序列化往返(落盘再读回)必须仍然通过校验。 + raw, err := json.Marshal(shuffled) + if err != nil { + t.Fatal(err) + } + var reloaded WriteSet + if err := json.Unmarshal(raw, &reloaded); err != nil { + t.Fatal(err) + } + if err := reloaded.Verify(); err != nil { + t.Fatalf("record read back from disk must verify: %v", err) + } +} + +func TestWriteSetVerifyRejectsTamperingAndMissingEvidence(t *testing.T) { + cases := map[string]func(*WriteSet){ + "内容被改": func(set *WriteSet) { + set.Rows[0].ID = "res-other" + }, + "换租户": func(set *WriteSet) { + set.TenantID = 20000 + }, + "整实例证据被清空": func(set *WriteSet) { + set.Universe = nil + }, + "缺预备份绑定": func(set *WriteSet) { + set.BackupSnapshotID = "" + }, + "缺摘要": func(set *WriteSet) { + set.Digest = "" + }, + "版本不认识": func(set *WriteSet) { + set.Version = 99 + }, + } + for name, mutate := range cases { + set := sampleWriteSet() + if err := set.Seal(); err != nil { + t.Fatal(err) + } + mutate(&set) + if err := set.Verify(); err == nil { + t.Fatalf("%s:必须校验失败", name) + } + } +} + +func TestWriteSetCreatedIDsOnlyCoversRowsCreatedByThisRelease(t *testing.T) { + set := sampleWriteSet() + if err := set.Seal(); err != nil { + t.Fatal(err) + } + created := set.CreatedIDs("resources") + if len(created) != 1 || created[0] != "res-new" { + t.Fatalf("created resources = %v, want [res-new](写前已存在的资源绝不能被当成残渣)", created) + } + if strings.Join(set.ExistingIDs("resources"), ",") != "res-old" { + t.Fatalf("existing resources = %v", set.ExistingIDs("resources")) + } + if set.UniverseDigest() == "" { + t.Fatal("universe digest must be derived from the whole-instance evidence") + } +} diff --git a/internal/releaseadapter/v071/library_export_integration_test.go b/internal/releaseadapter/v071/library_export_integration_test.go index 808244b..cb683ae 100644 --- a/internal/releaseadapter/v071/library_export_integration_test.go +++ b/internal/releaseadapter/v071/library_export_integration_test.go @@ -28,6 +28,11 @@ func TestLibraryExportIsolatedPostgres(t *testing.T) { t.Fatal(err) } defer db.Close() + // 同一个隔离库会被本包的多个用例复用:不清掉上一次的 schema 就会 + // "relation already exists",让人误以为是导出本身坏了。 + if _, err = db.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil { + t.Fatal(err) + } if _, err = db.Exec(ctx, libraryTestSchema); err != nil { t.Fatal(err) } diff --git a/internal/releaseadapter/v071/library_production_target.go b/internal/releaseadapter/v071/library_production_target.go index 6a52d58..d4eb824 100644 --- a/internal/releaseadapter/v071/library_production_target.go +++ b/internal/releaseadapter/v071/library_production_target.go @@ -395,9 +395,11 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc } sort.Strings(columns) // 本库行先删后插:除 resources(可能被多个知识库共享)外不需要 upsert。 + // resources 的 upsert 必须限定"同租户":同 ID 属于别的租户时整行覆盖会把 + // 别人的文件改坏(跨范围 ID 冲突已由写集合预检拒绝,这里是最后一道闸)。 if table == "resources" { statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb) -ON CONFLICT (id) DO UPDATE SET %s`, table, table, updateAssignments(columns)) +ON CONFLICT (id) DO UPDATE SET %s WHERE %s.tenant_id = $2`, table, table, updateAssignments(columns), table) } else { statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb)`, table, table) @@ -407,9 +409,19 @@ ON CONFLICT (id) DO UPDATE SET %s`, table, table, updateAssignments(columns)) if err != nil { return err } - if _, err := tx.Exec(ctx, statement, encoded); err != nil { + args := []any{encoded} + if table == "resources" { + args = append(args, t.scope.TenantID) + } + tag, err := tx.Exec(ctx, statement, args...) + if err != nil { return fmt.Errorf("写入生产 %s 失败: %w", table, err) } + if table == "resources" && tag.RowsAffected() != 1 { + var identity string + _ = json.Unmarshal(row["id"], &identity) + return fmt.Errorf("写入生产 resources 冲突:%s 属于其它租户,拒绝覆盖", identity) + } } return nil } @@ -663,67 +675,6 @@ func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrele return nil } -// dropOrphanResources 清掉"本次恢复后已无任何归属"的目标租户资源行。 -// -// 判据必须同时满足:属于目标租户、不在预备份快照里、没有任何 resource_binding、 -// 也没有任何分块正文以 resource:// 引用它。这样的行对目标应用完全不可见 -// (没有知识库能按它取到文件),只可能是失败写入留下的残渣。 -func (t *ProductionLibraryTarget) dropOrphanResources(ctx context.Context, backupSnapshotID string) error { - archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) - if err != nil { - return errors.New("归档不可用") - } - defer archive.Close() - manifest, err := archive.Load(backupSnapshotID) - if err != nil { - return errors.New("预备份快照不可读") - } - kept := []string{} - for _, part := range manifest.Parts { - if part.Kind != "database" || part.Name != "resources" { - continue - } - ids, err := archivedResourceIDs(archive, part) - if err != nil { - return err - } - kept = append(kept, ids...) - } - rows, err := t.pool.Query(ctx, ` -SELECT r.id FROM resources AS r -WHERE r.tenant_id=$1 - AND NOT (r.id = ANY($2::text[])) - AND NOT EXISTS (SELECT 1 FROM resource_bindings b WHERE b.resource_id=r.id) - AND NOT EXISTS (SELECT 1 FROM chunks c WHERE c.tenant_id=r.tenant_id AND c.deleted_at IS NULL - AND coalesce(r.handle,'') <> '' - AND position('resource://'||r.handle IN coalesce(c.content,'')) > 0)`, - t.scope.TenantID, kept) - if err != nil { - return fmt.Errorf("查找目标残留资源失败: %w", err) - } - orphans := make([]string, 0, 8) - for rows.Next() { - var id string - if err := rows.Scan(&id); err != nil { - rows.Close() - return err - } - orphans = append(orphans, id) - } - rows.Close() - if err := rows.Err(); err != nil { - return err - } - if len(orphans) == 0 { - return nil - } - if _, err := t.pool.Exec(ctx, `DELETE FROM resources WHERE tenant_id=$1 AND id = ANY($2::text[])`, - t.scope.TenantID, orphans); err != nil { - return fmt.Errorf("清理目标残留资源失败: %w", err) - } - return nil -} - // archivedResourceState 只核对"归档里列出的这些资源"是否还在且未变。 func (t *ProductionLibraryTarget) archivedResourceState(ctx context.Context, ids []string) (int64, string, error) { rows, err := t.pool.Query(ctx, `SELECT id FROM resources WHERE id = ANY($1::text[]) ORDER BY id`, ids) @@ -816,30 +767,59 @@ func archiveRowIdentity(table string, row map[string]json.RawMessage) string { } // Restore 用预备份把生产整库恢复回去:走同一条 Apply 路径,可重复执行。 +// +// 恢复必须带着"本次写集合 + 写前整实例证据"(写第一笔之前落盘、摘要登记在治理库)。 +// 缺它就直接失败:此刻目标已经变了,按现状推断"哪些是残渣"会删掉历史孤儿、 +// 同租户其它知识库暂未绑定的资源、被共享的资源——而预备份里没有这些行,删了回不来。 func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error { if backup.SnapshotID == "" { return errors.New("恢复需要绑定的预备份快照") } + if backup.WriteSet == nil { + return errors.New("恢复缺少持久化的写集合记录(作业/目标/预备份绑定):拒绝按目标现状推断残渣,维护保持") + } + if err := backup.WriteSet.Verify(); err != nil { + return fmt.Errorf("写集合记录校验失败,拒绝恢复:%w", err) + } + if backup.WriteSet.BackupSnapshotID != backup.SnapshotID { + return fmt.Errorf("写集合记录绑定的是预备份 %s,不是 %s", backup.WriteSet.BackupSnapshotID, backup.SnapshotID) + } + if backup.WriteSet.TenantID != t.scope.TenantID || backup.WriteSet.KnowledgeBaseID != t.scope.KnowledgeBaseID { + return fmt.Errorf("写集合记录绑定的是 %d/%s,与目标 %d/%s 不一致", + backup.WriteSet.TenantID, backup.WriteSet.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID) + } for _, stage := range libraryrelease.Stages() { if err := t.Apply(ctx, stage, backup.SnapshotID); err != nil { return fmt.Errorf("恢复阶段 %s 失败: %w", stage, err) } } - // resources 是多知识库共享表,回放阶段不删它:本次写入新增、恢复后已经 - // 没有任何归属的资源行会残留。它们对应用不可见,但属于"多写的"。 - if err := t.dropOrphanResources(ctx, backup.SnapshotID); err != nil { + // resources 是多知识库共享表,回放阶段不删它。这里只清"写前不存在 ∧ 由本次写入 ∧ + // 不在本次恢复内容里"的精确身份,历史孤儿与其它知识库的资源一律不动。 + archiveForCleanup, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) + if err != nil { + return errors.New("归档不可用") + } + restoredManifest, err := archiveForCleanup.Load(backup.SnapshotID) + if err != nil { + archiveForCleanup.Close() + return errors.New("预备份快照不可读") + } + restored, err := t.snapshotRowIdentities(archiveForCleanup, restoredManifest) + archiveForCleanup.Close() + if err != nil { return err } - // 恢复工具把四个阶段都跑完不算恢复完成:整实例必须回到预备份时刻的样子。 - // 只比目标范围会把"写到范围外"的残留当成已恢复(真实事故:恢复后仍多 67 行)。 - if len(backup.Universe) > 0 { - after, err := universeFingerprint(ctx, t.pool) - if err != nil { - return fmt.Errorf("恢复后读取整库指纹失败: %w", err) - } - if diff := universeDiff(backup.Universe, after); diff != "" { - return fmt.Errorf("恢复后目标与预备份时刻不一致:%s(很可能有写入逃出了预备份范围,不能当作已恢复)", diff) - } + if err := t.clearJobCreatedRows(ctx, *backup.WriteSet, restored); err != nil { + return err + } + // 恢复工具把四个阶段都跑完不算恢复完成:整实例(含其它知识库的内容)必须回到 + // 预备份时刻的样子。没有证据就不许降级跳过——那等于把"未经校验"报成"已恢复"。 + after, err := universeFingerprint(ctx, t.pool) + if err != nil { + return fmt.Errorf("恢复后读取整库指纹失败: %w", err) + } + if diff := universeDiff(backup.WriteSet.Universe, after); diff != "" { + return fmt.Errorf("恢复后目标与预备份时刻不一致:%s(很可能有写入逃出了预备份范围,不能当作已恢复)", diff) } return nil } diff --git a/internal/releaseadapter/v071/library_production_target_integration_test.go b/internal/releaseadapter/v071/library_production_target_integration_test.go index 51c8ed0..4d47bfd 100644 --- a/internal/releaseadapter/v071/library_production_target_integration_test.go +++ b/internal/releaseadapter/v071/library_production_target_integration_test.go @@ -108,6 +108,30 @@ func TestLibraryProductionTargetIsolated(t *testing.T) { t.Fatalf("target: %v", err) } defer target.Close() + // 写集合在生产路径上是"写第一笔之前"算好的;这个用例直接调 Apply,所以在这里算, + // 整实例证据就是此刻(任何写入之前)的目标状态。 + // 夹具里那条软删除文档不在本快照里(快照只含 doc-safe/chunk-safe/wiki-safe), + // 恢复按目标范围回放会正确地把它清掉;为了让"整实例证据"与恢复结果可比,先删掉它。 + // 同时把夹具行的内容对齐到本快照(快照是可读的完整版本形态):整实例内容证据 + // 才可能与"恢复完成后的目标"逐字节一致。 + for _, statement := range []string{ + `DELETE FROM knowledges WHERE id='deleted-doc'`, + `UPDATE knowledges SET file_path='' WHERE id='doc-safe'`, + `UPDATE wiki_pages SET content='wiki', status=NULL WHERE id='wiki-safe'`, + // 快照里这些表是空的:目标范围先与它对齐,否则"整实例内容证据"比的是 + // 快照里不存在的行,恢复(正确地)删掉它们后反而被判不一致。 + `DELETE FROM resource_bindings`, + `DELETE FROM resources`, + `DELETE FROM wiki_page_issues`, + `DELETE FROM wiki_log_entries`, + `DELETE FROM knowledge_tags`, + `DELETE FROM knowledge_tag_relations`, + } { + if _, err := pool.Exec(ctx, statement); err != nil { + t.Fatal(err) + } + } + initialLedger := planLedgerFor(t, target, first, first) for _, stage := range []libraryrelease.Stage{libraryrelease.StageDatabase, libraryrelease.StageWiki, libraryrelease.StageIndex} { if err := target.Apply(ctx, stage, first); err != nil { @@ -149,7 +173,8 @@ func TestLibraryProductionTargetIsolated(t *testing.T) { t.Fatalf("cross-scope rows must survive, got %d", others) } // 恢复:把第一个快照再应用一次,撤销的内容回来。 - if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup", SnapshotID: first}); err != nil { + // 恢复必须带着写集合记录(生产路径在写第一笔之前就算好并落盘)。 + if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup", SnapshotID: first, WriteSet: initialLedger}); err != nil { t.Fatalf("restore: %v", err) } if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: first}); err != nil { @@ -193,6 +218,9 @@ func TestLibraryProductionTargetIsolated(t *testing.T) { if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: first}); err == nil { t.Fatal("verify must detect a missing chunk row") } + // 这次恢复要用"预备份时刻"的整实例证据(Prepare 已经返回它),不是当前状态: + // 目标刚被故意破坏过,按现状算出来的证据会与恢复结果对不上。 + backup.WriteSet = planLedgerWithUniverse(t, target, backup.SnapshotID, backup.SnapshotID, backup.Universe) if err := target.Restore(ctx, backup); err != nil { t.Fatalf("restore from backup: %v", err) } @@ -298,7 +326,7 @@ func TestLibraryReleaseRunIsolated(t *testing.T) { t.Fatal(err) } now := time.Now().UTC() - request := libraryrelease.Request{Target: "internal", SnapshotID: snapshotID, + request := libraryrelease.Request{JobID: "job-e2e-isolated", Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-e2e", Now: now, Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-e2e", ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}} @@ -561,7 +589,7 @@ func TestLibraryCrossScopeSyncRollbackLeavesNoResidueIsolated(t *testing.T) { t.Fatalf("backup store: %v", err) } now := time.Now().UTC() - request := libraryrelease.Request{Target: "internal", SnapshotID: snapshotID, + request := libraryrelease.Request{JobID: "job-cross-scope", Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-cross-scope", Now: now, Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-cross-scope", ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}} @@ -729,3 +757,29 @@ func sealCrossScopeFixture(t *testing.T, archive *librarysnapshot.Archive, sourc } func formatTenant(tenant uint64) string { return strconv.FormatUint(tenant, 10) } + +// planLedgerFor 给"直接调 Apply 再 Restore"的用例造一份真实写集合: +// 生产路径由 libraryrelease.Run 在写第一笔之前完成同样的动作。 +func planLedgerFor(t *testing.T, target *ProductionLibraryTarget, snapshotID, backupID string) *libraryrelease.WriteSet { + t.Helper() + return planLedgerWithUniverse(t, target, snapshotID, backupID, nil) +} + +func planLedgerWithUniverse(t *testing.T, target *ProductionLibraryTarget, snapshotID, backupID string, + universe map[string]libraryrelease.TableFingerprint) *libraryrelease.WriteSet { + t.Helper() + ctx := context.Background() + if universe == nil { + computed, err := universeFingerprint(ctx, target.Pool()) + if err != nil { + t.Fatalf("whole-instance fingerprint: %v", err) + } + universe = computed + } + set, err := target.PlanWriteSet(ctx, libraryrelease.WriteSetPlan{JobID: "job-test-" + snapshotID[:8], + Target: "internal", BackupSnapshotID: backupID, TargetSnapshotID: snapshotID, Universe: universe}) + if err != nil { + t.Fatalf("plan write set: %v", err) + } + return &set +} diff --git a/internal/releaseadapter/v071/library_universe.go b/internal/releaseadapter/v071/library_universe.go index 4abf5fb..08f8458 100644 --- a/internal/releaseadapter/v071/library_universe.go +++ b/internal/releaseadapter/v071/library_universe.go @@ -13,19 +13,23 @@ import ( "knowledge-governance/internal/librarysnapshot" ) -// universeFingerprint 是"整实例"的知识域指纹:不限租户/知识库的行数 + 身份集合摘要。 +// universeFingerprint 是"整实例"的知识域指纹:不限租户/知识库的行数与**逐行内容**摘要。 // // 为什么不能只比目标范围:恢复只按目标范围删除/回放,写到范围外的残留它看不见 // (真实事故:作业报"已恢复到预备份",目标却多出 4 文件 / 38 分块 / 21 资源 / -// 4 绑定)。整实例指纹让"多写了什么"必然暴露,同时也能证明恢复没有误删别的 -// 知识库的行。 +// 4 绑定)。整实例指纹让"多写了什么"必然暴露,同时也能证明恢复没有误删、误改 +// 别的知识库的行。 +// +// 为什么必须是内容而不是身份集合:只比 id 集合时,"另一个知识库的同 ID 行被覆盖 +// 写坏"(资源表的 ON CONFLICT 就会这样)在指纹上完全看不出来——数量、身份都一样。 +// 因此摘要取自每行 to_jsonb 的 md5,再按身份排序聚合。 func universeFingerprint(ctx context.Context, pool *pgxpool.Pool) (map[string]libraryrelease.TableFingerprint, error) { result := make(map[string]libraryrelease.TableFingerprint, len(productionLibraryTables)) for _, table := range productionLibraryTables { identity := identityExpr(table) statement := fmt.Sprintf( - `SELECT count(*), coalesce(md5(string_agg((%s)::text, E'\n' ORDER BY (%s)::text)), '') FROM %s`, - identity, identity, table) + `SELECT count(*), coalesce(md5(string_agg(md5(to_jsonb(x)::text), E'\n' ORDER BY (%s)::text)), '') FROM %s AS x`, + identity, table) var fingerprint libraryrelease.TableFingerprint if err := pool.QueryRow(ctx, statement).Scan(&fingerprint.Count, &fingerprint.Digest); err != nil { return nil, fmt.Errorf("读取整库 %s 指纹失败: %w", table, err) diff --git a/internal/releaseadapter/v071/library_writeset.go b/internal/releaseadapter/v071/library_writeset.go new file mode 100644 index 0000000..faeaa0e --- /dev/null +++ b/internal/releaseadapter/v071/library_writeset.go @@ -0,0 +1,382 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +// 写集合台账:一次发布的"明确写集合 + 写前存在状态 + 预备份时刻整实例证据"。 +// +// 为什么必须是持久记录而不是恢复时现算: +// 恢复发生在写失败之后,目标已经变了。此刻按"目标里没人引用的资源就是残渣"去删, +// 会连历史孤儿、同租户其它知识库暂未绑定的资源一起删掉——而预备份里没有这些行, +// 删了就是不可恢复的数据丢失(真实缺陷)。唯一可靠的口径是写之前就记下 +// "这次要写哪些精确 ID、哪些写前就已存在",恢复只删"写前不存在 ∧ 由本次写入"的那些。 + +// writeSetDirName 是台账在归档根下的固定目录,与快照同级:同一块持久卷, +// 重启后按摘要直接取回。 +const writeSetDirName = "writesets" + +func writeSetPath(archiveRoot, digest string) string { + return filepath.Join(archiveRoot, writeSetDirName, digest+".json") +} + +// PersistWriteSet 把台账按内容寻址落盘(文件名即 sha256),随后只读。 +// 幂等:同摘要文件已存在且内容一致就直接返回;内容不一致说明摘要被占用,必须失败。 +func PersistWriteSet(archiveRoot string, set *libraryrelease.WriteSet) (string, error) { + if set.Digest == "" { + if err := set.Seal(); err != nil { + return "", err + } + } + if len(set.Universe) == 0 { + return "", errors.New("写集合缺少整实例内容证据,拒绝落盘") + } + canonical, err := set.CanonicalBytes() + if err != nil { + return "", err + } + sum := sha256.Sum256(canonical) + digest := hex.EncodeToString(sum[:]) + if digest != set.Digest { + return "", errors.New("写集合摘要与内容不符,拒绝落盘") + } + raw, err := json.Marshal(*set) + if err != nil { + return "", fmt.Errorf("编码写集合失败: %w", err) + } + directory := filepath.Join(archiveRoot, writeSetDirName) + if err := os.MkdirAll(directory, 0700); err != nil { + return "", fmt.Errorf("创建写集合目录失败: %w", err) + } + path := writeSetPath(archiveRoot, digest) + if existing, readErr := os.ReadFile(path); readErr == nil { + var recorded libraryrelease.WriteSet + if json.Unmarshal(existing, &recorded) != nil || recorded.Verify() != nil || recorded.Digest != digest { + return "", errors.New("写集合记录同名但内容不符,拒绝覆盖") + } + return path, nil + } else if !errors.Is(readErr, os.ErrNotExist) { + return "", fmt.Errorf("读取已有写集合失败: %w", readErr) + } + temporary := path + ".tmp" + file, err := os.OpenFile(temporary, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return "", fmt.Errorf("写入写集合失败: %w", err) + } + if _, err := file.Write(raw); err != nil { + file.Close() + os.Remove(temporary) + return "", fmt.Errorf("写入写集合失败: %w", err) + } + if err := file.Sync(); err != nil { + file.Close() + os.Remove(temporary) + return "", fmt.Errorf("写集合未落盘: %w", err) + } + if err := file.Close(); err != nil { + os.Remove(temporary) + return "", fmt.Errorf("写集合未落盘: %w", err) + } + if err := os.Chmod(temporary, 0444); err != nil { + os.Remove(temporary) + return "", fmt.Errorf("写集合只读化失败: %w", err) + } + if err := os.Rename(temporary, path); err != nil { + os.Remove(temporary) + return "", fmt.Errorf("写集合落盘失败: %w", err) + } + return path, nil +} + +// LoadWriteSet 按摘要取回台账:文件名、字节摘要、内容摘要三者必须一致。 +// 任何一步不符都返回错误——治理库里的引用指向的就是这个摘要。 +func LoadWriteSet(archiveRoot, digest string) (*libraryrelease.WriteSet, error) { + if digest == "" { + return nil, errors.New("没有写集合摘要,无法取回记录") + } + path := writeSetPath(archiveRoot, digest) + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("写集合记录缺失(%s)", digest) + } + var set libraryrelease.WriteSet + if err := json.Unmarshal(raw, &set); err != nil { + return nil, errors.New("写集合记录不可解析") + } + if err := set.Verify(); err != nil { + return nil, err + } + if set.Digest != digest { + return nil, errors.New("写集合记录摘要不符") + } + return &set, nil +} + +// PlanWriteSet 在写第一笔之前算出"这次明确要写什么",拒绝跨范围 ID 冲突, +// 并把台账落盘。返回的记录已 Seal 过;调用方负责把摘要登记到治理库。 +func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) { + if plan.JobID == "" || plan.BackupSnapshotID == "" || plan.TargetSnapshotID == "" { + return libraryrelease.WriteSet{}, errors.New("写集合缺少作业/预备份/快照身份") + } + if len(plan.Universe) == 0 { + return libraryrelease.WriteSet{}, errors.New("预备份没有整实例内容证据,拒绝在没有保护证据的情况下写目标") + } + archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) + if err != nil { + return libraryrelease.WriteSet{}, errors.New("归档不可用") + } + defer archive.Close() + manifest, err := archive.Load(plan.TargetSnapshotID) + if err != nil { + return libraryrelease.WriteSet{}, errors.New("要同步的快照不可读") + } + if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil { + return libraryrelease.WriteSet{}, err + } + backupResources, err := archivedSnapshotResourceIDs(archive, plan.BackupSnapshotID) + if err != nil { + return libraryrelease.WriteSet{}, err + } + set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: plan.JobID, Target: plan.Target, + TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID, + BackupSnapshotID: plan.BackupSnapshotID, TargetSnapshotID: plan.TargetSnapshotID, + CreatedAt: t.now().UTC(), Universe: plan.Universe} + for _, part := range manifest.Parts { + if part.Kind != "database" && part.Kind != "wiki" { + continue + } + ids, err := archivedRowIdentities(archive, part) + if err != nil { + return libraryrelease.WriteSet{}, err + } + if len(ids) == 0 { + continue + } + owners, err := t.existingRowOwners(ctx, part.Name, ids) + if err != nil { + return libraryrelease.WriteSet{}, err + } + for _, id := range ids { + owner, exists := owners[id] + if !exists { + set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id}) + continue + } + if err := t.assertWriteSetOwnership(part.Name, id, owner, backupResources); err != nil { + return libraryrelease.WriteSet{}, err + } + set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id, Existing: true, Owner: owner}) + } + } + if err := set.Seal(); err != nil { + return libraryrelease.WriteSet{}, err + } + if _, err := PersistWriteSet(t.config.ArchiveRoot, &set); err != nil { + return libraryrelease.WriteSet{}, err + } + return set, nil +} + +// archivedSnapshotResourceIDs 读一份快照里"属于目标自己的资源身份"。 +// 资源表被多个知识库共享,只有在这份预备份里的资源行才允许被覆盖写。 +func archivedSnapshotResourceIDs(archive *librarysnapshot.Archive, snapshotID string) (map[string]bool, error) { + ids := map[string]bool{} + if snapshotID == "" { + return ids, nil + } + manifest, err := archive.Load(snapshotID) + if err != nil { + return nil, errors.New("预备份快照不可读") + } + for _, part := range manifest.Parts { + if part.Kind != "database" || part.Name != "resources" { + continue + } + values, err := archivedResourceIDs(archive, part) + if err != nil { + return nil, err + } + for _, id := range values { + ids[id] = true + } + } + return ids, nil +} + +// existingRowOwners 读出这些身份在目标里的写前归属("/")。 +// 没有 id 列的关系表按父表 knowledges 判归属。 +func (t *ProductionLibraryTarget) existingRowOwners(ctx context.Context, table string, ids []string) (map[string]string, error) { + owners := map[string]string{} + if table == "knowledge_tag_relations" { + rows, err := t.pool.Query(ctx, ` +SELECT r.knowledge_id, coalesce(r.tag_id,''), coalesce(k.tenant_id,0), coalesce(k.knowledge_base_id,'') +FROM knowledge_tag_relations r LEFT JOIN knowledges k ON k.id=r.knowledge_id +WHERE r.knowledge_id = ANY($1::text[])`, ids) + if err != nil { + return nil, fmt.Errorf("读取目标 %s 归属失败", table) + } + defer rows.Close() + for rows.Next() { + var knowledge, tag, kb string + var tenant uint64 + if err := rows.Scan(&knowledge, &tag, &tenant, &kb); err != nil { + return nil, err + } + owners[knowledge+":"+tag] = fmt.Sprintf("%d/%s", tenant, kb) + } + return owners, rows.Err() + } + statement := fmt.Sprintf(`SELECT id, tenant_id, coalesce(knowledge_base_id,'') FROM %s WHERE id = ANY($1::text[])`, table) + if table == "resource_bindings" { + // 绑定表没有 knowledge_base_id:归属只能走父表,否则"同租户"就会被当成"自己的"。 + statement = `SELECT b.id, b.tenant_id, + coalesce(k.knowledge_base_id, c.knowledge_base_id, '') +FROM resource_bindings b +LEFT JOIN knowledges k ON b.owner_type='knowledge' AND k.id=b.owner_id +LEFT JOIN chunks c ON b.owner_type='chunk' AND c.id=b.owner_id +WHERE b.id = ANY($1::text[])` + } else if !tablesWithKB[table] { + statement = fmt.Sprintf(`SELECT id, tenant_id, '' FROM %s WHERE id = ANY($1::text[])`, table) + } + rows, err := t.pool.Query(ctx, statement, ids) + if err != nil { + return nil, fmt.Errorf("读取目标 %s 归属失败", table) + } + defer rows.Close() + for rows.Next() { + var id, kb string + var tenant uint64 + if err := rows.Scan(&id, &tenant, &kb); err != nil { + return nil, err + } + owners[id] = fmt.Sprintf("%d/%s", tenant, kb) + } + return owners, rows.Err() +} + +// assertWriteSetOwnership 是跨范围 ID 冲突的预检:同 ID 已经是别人的行时, +// 写入会改掉别人的数据(resources 尤其危险:ON CONFLICT 会整行覆盖)。 +// 宁可整次拒绝,也不许"覆盖一下看起来也能跑"。 +func (t *ProductionLibraryTarget) assertWriteSetOwnership(table, id, owner string, backupResources map[string]bool) error { + target := fmt.Sprintf("%d/%s", t.scope.TenantID, t.scope.KnowledgeBaseID) + if table == "resources" { + // 资源只有租户:同租户不等于同知识库,必须落在预备份范围内才算自己的。 + if owner != fmt.Sprintf("%d/", t.scope.TenantID) { + return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner) + } + if !backupResources[id] { + return fmt.Errorf("资源 ID 冲突:%s 存在于目标租户但不在预备份范围内(属于其它知识库),拒绝覆盖", id) + } + return nil + } + if owner != target && owner != fmt.Sprintf("%d/", t.scope.TenantID) { + return fmt.Errorf("%s ID 冲突:%s 已属于 %s,拒绝写入", table, id, owner) + } + return nil +} + +// clearJobCreatedRows 只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复快照里"的精确身份, +// 并且对资源额外要求它仍然没有任何绑定/正文引用(作业失败期间被别的知识库接管 +// 使用的资源不能删)。 +// +// 为什么要减掉"快照里的身份":这份台账可能是为另一次动作算的(例如手工从某份 +// 预备份修复目标)。只要这一行就是恢复要回到的内容,它就不是残渣——按台账删掉 +// 等于把刚恢复的行再删一次。 +func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set libraryrelease.WriteSet, + restored map[string]map[string]bool) error { + // 子表先删:resource_bindings → chunks/knowledges、wiki 子表 → wiki 页 → 资源。 + for _, table := range []string{"resource_bindings", "chunks", "knowledges", "knowledge_tag_relations", + "wiki_page_issues", "wiki_log_entries", "wiki_pages", "wiki_folders", "knowledge_tags", "resources"} { + ids := make([]string, 0, 8) + for _, id := range set.CreatedIDs(table) { + if restored[table][id] { + continue + } + ids = append(ids, id) + } + if len(ids) == 0 { + continue + } + statement := fmt.Sprintf(`DELETE FROM %s WHERE (%s)::text = ANY($1::text[])`, table, identityExpr(table)) + if table == "resources" { + statement += ` + AND NOT EXISTS (SELECT 1 FROM resource_bindings b WHERE b.resource_id=resources.id) + AND NOT EXISTS (SELECT 1 FROM chunks c WHERE c.deleted_at IS NULL + AND coalesce(resources.handle,'') <> '' + AND position('resource://'||resources.handle IN coalesce(c.content,'')) > 0)` + } + if _, err := t.pool.Exec(ctx, statement, ids); err != nil { + return fmt.Errorf("清理本次新增的 %s 失败: %w", table, err) + } + } + // 复核:本次新建的行一个都不许剩。资源若因"被引用"而保留,那说明恢复没回到 + // 预备份状态,必须报出来让维护保持,而不是静默放过。 + leftovers := make([]string, 0, 4) + for _, table := range productionLibraryTables { + ids := make([]string, 0, 8) + for _, id := range set.CreatedIDs(table) { + if restored[table][id] { + continue + } + ids = append(ids, id) + } + if len(ids) == 0 { + continue + } + statement := fmt.Sprintf(`SELECT (%s)::text FROM %s WHERE (%s)::text = ANY($1::text[])`, + identityExpr(table), table, identityExpr(table)) + rows, err := t.pool.Query(ctx, statement, ids) + if err != nil { + return fmt.Errorf("复核本次新增的 %s 失败: %w", table, err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return err + } + leftovers = append(leftovers, table+"/"+id) + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + } + if len(leftovers) > 0 { + sort.Strings(leftovers) + return fmt.Errorf("恢复后仍留下本次写入的 %d 行(%v):目标未回到预备份状态,不能当作已恢复", + len(leftovers), leftovers) + } + return nil +} + +// snapshotRowIdentities 读出"恢复要用这份快照"的全部行身份(按表)。 +func (t *ProductionLibraryTarget) snapshotRowIdentities(archive *librarysnapshot.Archive, + manifest librarysnapshot.Manifest) (map[string]map[string]bool, error) { + identities := map[string]map[string]bool{} + for _, part := range manifest.Parts { + if part.Kind != "database" && part.Kind != "wiki" { + continue + } + ids, err := archivedRowIdentities(archive, part) + if err != nil { + return nil, err + } + identities[part.Name] = map[string]bool{} + for _, id := range ids { + identities[part.Name][id] = true + } + } + return identities, nil +} diff --git a/internal/releaseadapter/v071/library_writeset_integration_test.go b/internal/releaseadapter/v071/library_writeset_integration_test.go new file mode 100644 index 0000000..9757bec --- /dev/null +++ b/internal/releaseadapter/v071/library_writeset_integration_test.go @@ -0,0 +1,685 @@ +package v071 + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "sort" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +// 这一组用例验证的是"失败恢复不许误删目标自己的数据"。 +// +// 资源用真实对象存储(MinIO/S3),PostgreSQL 与 Elasticsearch 也是真服务: +// 只有"写失败"是按需注入的(flakyStore),业务不变量不靠桩对象成立。 +// +// 为什么必须真实对象存储:旧实现按"目标租户里没有绑定、也没被正文引用"清理资源行, +// 它在内存桩上与真实存储上表现一样(都会删),但资源字节/定位符的改写只有在真实 +// bucket 上才能证明"恢复后仍是同一份内容"。桩只用于单元级。 + +type writeSetFixture struct { + ctx context.Context + pool *pgxpool.Pool + client *http.Client + searchBase string + index string + archiveRoot string + objects *flakyStore + bucket string +} + +const ( + wsTargetTenant = 10001 + wsTargetKB = "kb-target" + wsOtherKB = "kb-other" + wsSourceTenant = 10002 + wsSourceKB = "kb-source" + wsForeignTnt = 20000 + wsBucket = "kg-wire-test-bucket" + + wsHandleTarget = "aaaaaaaaaaaaaaaaaaaaaa" + wsHandleShared = "bbbbbbbbbbbbbbbbbbbbbb" + wsHandleOther = "cccccccccccccccccccccc" + wsHandleOrphan = "dddddddddddddddddddddd" + wsHandleSource = "eeeeeeeeeeeeeeeeeeeeee" + wsHandleForeign = "ffffffffffffffffffffff" +) + +func wsLocator(backend, provider, bucket string, tenant uint64, id string) string { + return fmt.Sprintf("storage://%s/%s://%s/%d/%s/a.png", backend, provider, bucket, tenant, id) +} + +// flakyStore 在真实 S3 之上按需注入写失败:只让"新对象上传"失败, +// 已存在的对象仍可覆盖(恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏)。 +// Open/Stat/Copy 全部透传。 +type flakyStore struct { + s3 *S3ObjectStore + failPut atomic.Bool + // failPath 是注入失败的定位符片段:只让"这次要新增的那一个资源"上传失败, + // 恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏。 + failPath string + puts atomic.Int64 +} + +func (s *flakyStore) Put(ctx context.Context, provider, path string, reader io.Reader, size int64) error { + if s.failPut.Load() && (s.failPath == "" || strings.Contains(path, s.failPath)) { + return errors.New("注入的对象写入失败") + } + s.puts.Add(1) + return s.s3.Put(ctx, provider, path, reader, size) +} + +func (s *flakyStore) Stat(ctx context.Context, provider, path string) (int64, string, error) { + return s.s3.Stat(ctx, provider, path) +} + +func (s *flakyStore) Copy(ctx context.Context, provider, source, target string) error { + return s.s3.Copy(ctx, provider, source, target) +} + +func (s *flakyStore) Open(ctx context.Context, provider, path string) (io.ReadCloser, error) { + return s.s3.Open(ctx, provider, path) +} + +func openWriteSetFixture(t *testing.T) *writeSetFixture { + t.Helper() + return openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true}) +} + +// writeSetFixtureOptions 让"被杀掉的子进程"复用父进程已经建好的现场: +// 不重建 schema、不新建索引,也不换归档根(写集合与预备份都落在那一个根下)。 +type writeSetFixtureOptions struct { + reset bool + archiveRoot string + index string +} + +func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writeSetFixture { + t.Helper() + dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL") + base := os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL") + endpoint := os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT") + access, secret := os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"), os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY") + if dsn == "" || base == "" || endpoint == "" || access == "" || secret == "" || + os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" { + t.Skip("requires isolated PG, ES and object storage opt-in") + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") { + t.Fatal("isolated database required") + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) + t.Cleanup(cancel) + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatalf("open isolated PG: %v", err) + } + t.Cleanup(pool.Close) + if options.reset { + if _, err := pool.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil { + t.Fatalf("reset isolated schema: %v", err) + } + } + schemaStatements := []string{ + libraryTestSchema, + // 清掉共用夹具里的示例行:本文件只用自己造的目标/其它租户数据, + // 免得"目标非空"被别的用例的行弄成不确定。 + `DELETE FROM knowledges; DELETE FROM chunks; DELETE FROM knowledge_tags; + DELETE FROM knowledge_tag_relations; DELETE FROM resources; + DELETE FROM resource_bindings; DELETE FROM wiki_pages; DELETE FROM wiki_folders; + DELETE FROM wiki_page_issues; DELETE FROM wiki_log_entries`, + `ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published'`, + `ALTER TABLE knowledge_tag_relations ADD COLUMN tenant_id bigint`, + `ALTER TABLE resources ADD COLUMN provider text, ADD COLUMN size bigint, ADD COLUMN storage_backend_id text`, + `ALTER TABLE resources ADD PRIMARY KEY (id)`, + `ALTER TABLE resource_bindings ADD COLUMN relation text`, + `CREATE UNIQUE INDEX wiki_pages_scope_slug ON wiki_pages(tenant_id, knowledge_base_id, slug)`, + `CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`, + `INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + wsBucket + `"}',now())`, + `INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`, + } + if options.reset { + for _, statement := range schemaStatements { + if _, err := pool.Exec(ctx, statement); err != nil { + t.Fatalf("fixture schema: %v", err) + } + } + } + + token := make([]byte, 8) + if _, err := rand.Read(token); err != nil { + t.Fatal(err) + } + index := "kg_library_test_" + hex.EncodeToString(token) + client := &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }} + call := func(method, path, body string, want int) { + t.Helper() + request, err := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Content-Type", "application/json") + response, err := client.Do(request) + if err != nil { + t.Fatalf("fixture HTTP: %v", err) + } + defer response.Body.Close() + if response.StatusCode != want { + t.Fatalf("fixture %s %s => %d, want %d", method, path, response.StatusCode, want) + } + } + if options.index != "" { + index = options.index + } + if options.reset { + call("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"},"chunk_id":{"type":"keyword"},"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":2}}}}`, 200) + } + if options.reset { + t.Cleanup(func() { + call("DELETE", "/"+index, "", 200) + call("HEAD", "/"+index, "", 404) + }) + } + + root := options.archiveRoot + if root == "" { + root = t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + } + store, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: endpoint, + Region: "us-east-1", AccessKeyID: access, SecretAccessKey: secret, Bucket: wsBucket, + ForcePathStyle: true, MaxObjectBytes: 64 * 1024 * 1024}) + if err != nil { + t.Fatalf("object store: %v", err) + } + if err := store.EnsureBucket(ctx); err != nil { + t.Fatalf("ensure bucket: %v", err) + } + return &writeSetFixture{ctx: ctx, pool: pool, client: client, searchBase: base, index: index, + archiveRoot: root, objects: &flakyStore{s3: store}, bucket: wsBucket} +} + +// seedTargetData 造"目标本来就有数据"的现场:目标自己的行、同租户另一个知识库的行(含资源)、 +// 被两个知识库共享的资源、一个历史孤儿资源、另一个租户的行。 +func seedTargetData(t *testing.T, fx *writeSetFixture) { + t.Helper() + ctx := fx.ctx + locator := func(backend string, tenant uint64, id string) string { + return wsLocator(backend, "minio", wsBucket, tenant, id) + } + for _, object := range []struct { + tenant uint64 + id string + }{ + {10001, "res-target"}, {10001, "res-shared"}, {10001, "res-other"}, + {10001, "res-orphan"}, {20000, "res-foreign"}, + } { + path := locator("backend-target", object.tenant, object.id) + if err := fx.objects.s3.Put(ctx, "minio", path, bytes.NewReader([]byte("12345678")), 8); err != nil { + t.Fatalf("seed object %s: %v", object.id, err) + } + } + statements := []string{ + // 目标自己:文档 + 分块(正文引用目标资源)+ 资源 + 绑定 + Wiki。 + `INSERT INTO knowledges VALUES('doc-target',10001,'` + wsTargetKB + `','completed',0,'',NULL)`, + `INSERT INTO chunks VALUES('chunk-target',10001,'` + wsTargetKB + `','doc-target','图:resource://` + wsHandleTarget + `',NULL)`, + `INSERT INTO resources VALUES('res-target','` + wsHandleTarget + `',10001,'` + locator("backend-target", 10001, "res-target") + `',NULL,'minio',8,'backend-target')`, + `INSERT INTO resource_bindings VALUES('bind-target',10001,'res-target','knowledge','doc-target',NULL)`, + `INSERT INTO wiki_pages VALUES('wiki-target',10001,'` + wsTargetKB + `','index','目标 wiki',NULL,'published')`, + // 同租户另一个知识库:数据与资源都必须原样保留。 + `INSERT INTO knowledges VALUES('doc-other',10001,'` + wsOtherKB + `','completed',0,'',NULL)`, + `INSERT INTO chunks VALUES('chunk-other',10001,'` + wsOtherKB + `','doc-other','别的知识库正文',NULL)`, + `INSERT INTO resources VALUES('res-other','` + wsHandleOther + `',10001,'` + locator("backend-target", 10001, "res-other") + `',NULL,'minio',8,'backend-target')`, + `INSERT INTO resource_bindings VALUES('bind-other',10001,'res-other','knowledge','doc-other',NULL)`, + // 共享资源:同时被目标知识库与另一个知识库绑定。 + `INSERT INTO resources VALUES('res-shared','` + wsHandleShared + `',10001,'` + locator("backend-target", 10001, "res-shared") + `',NULL,'minio',8,'backend-target')`, + `INSERT INTO resource_bindings VALUES('bind-shared-target',10001,'res-shared','knowledge','doc-target',NULL)`, + `INSERT INTO resource_bindings VALUES('bind-shared-other',10001,'res-shared','knowledge','doc-other',NULL)`, + // 历史孤儿:没有任何绑定、也没有正文引用。它不是这次作业写的,绝不能被删。 + `INSERT INTO resources VALUES('res-orphan','` + wsHandleOrphan + `',10001,'` + locator("backend-target", 10001, "res-orphan") + `',NULL,'minio',8,'backend-target')`, + // 另一个租户的行。 + `INSERT INTO knowledges VALUES('doc-foreign',20000,'kb-foreign','completed',0,'',NULL)`, + `INSERT INTO chunks VALUES('chunk-foreign',20000,'kb-foreign','doc-foreign','外部正文',NULL)`, + `INSERT INTO resources VALUES('res-foreign','` + wsHandleForeign + `',20000,'` + locator("backend-target", 20000, "res-foreign") + `',NULL,'minio',8,'backend-target')`, + `INSERT INTO resource_bindings VALUES('bind-foreign',20000,'res-foreign','knowledge','doc-foreign',NULL)`, + } + for _, statement := range statements { + if _, err := fx.pool.Exec(ctx, statement); err != nil { + t.Fatalf("seed target: %v\n%s", err, statement) + } + } + // 目标索引自洽:目标分块必须有对应文档,否则预备份会(正确地)拒绝。 + fx.indexDoc(t, "chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "图:"+wsHandleTarget) +} + +func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb, knowledge, content string) { + t.Helper() + body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":[0.1,0.2]}`, + tenant, kb, knowledge, id, id, content) + request, err := http.NewRequestWithContext(fx.ctx, "PUT", fmt.Sprintf("%s/%s/_doc/%s?refresh=true", fx.searchBase, fx.index, id), + strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Content-Type", "application/json") + response, err := fx.client.Do(request) + if err != nil { + t.Fatalf("index doc: %v", err) + } + defer response.Body.Close() + if response.StatusCode != 201 && response.StatusCode != 200 { + t.Fatalf("index doc %s => %d", id, response.StatusCode) + } +} + +// sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。 +func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string { + t.Helper() + archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024) + if err != nil { + t.Fatal(err) + } + defer archive.Close() + encode := func(value any) []byte { + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return encoded + } + parts := map[string][]map[string]any{ + "knowledges": {{"id": "doc-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, + "parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}}, + "chunks": {{"id": "chunk-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, + "knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}}, + "knowledge_tags": {}, + "knowledge_tag_relations": {}, + "resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": wsSourceTenant, + "storage_backend_id": "backend-source", "provider": "oss", + "physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-src"), + "size": int64(8)}}, + "resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": wsSourceTenant, + "owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}}, + "wiki_folders": {}, + "wiki_pages": {{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"}}, + "wiki_page_issues": {}, + "wiki_log_entries": {}, + } + if tweak != nil { + tweak(parts) + } + kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database", + "knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database", + "wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"} + manifestParts := make([]librarysnapshot.Part, 0, 14) + names := make([]string, 0, len(parts)) + for name := range parts { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + part, err := archive.Put(bytes.NewReader(encode(parts[name]))) + if err != nil { + t.Fatal(err) + } + part.Kind, part.Name, part.Count = kinds[name], name, int64(len(parts[name])) + manifestParts = append(manifestParts, part) + } + resourcePart, err := archive.Put(bytes.NewReader([]byte("12345678"))) + if err != nil { + t.Fatal(err) + } + resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1 + manifestParts = append(manifestParts, resourcePart) + indexPart, err := archive.Put(bytes.NewReader(encode([]map[string]any{{"id": "chunk-src", "source": map[string]any{ + "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src", + "chunk_id": "chunk-src", "source_id": "chunk-src", "content": "来源正文 resource://" + wsHandleSource, + "embedding": []float64{0.1, 0.2}}}}))) + if err != nil { + t.Fatal(err) + } + indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", 1 + manifestParts = append(manifestParts, indexPart) + manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy, + Format: librarysnapshot.Format, Scope: "internal", TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB, + CreatedAt: time.Now().UTC(), Parts: manifestParts, + Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, + RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4", + EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}} + id, err := archive.Seal(manifest) + if err != nil { + t.Fatalf("seal source snapshot: %v", err) + } + return id +} + +func (fx *writeSetFixture) target(t *testing.T) *ProductionLibraryTarget { + t.Helper() + target, err := NewProductionLibraryTarget(fx.ctx, ProductionLibraryConfig{ + Postgres: os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), SearchURL: fx.searchBase, SearchIndex: fx.index, + Objects: fx.objects, ObjectsReader: fx.objects, ArchiveRoot: fx.archiveRoot, + ObjectsBucket: fx.bucket, MaxDocuments: 100, HTTPClient: fx.client, + }, ProductionScope{TenantID: wsTargetTenant, KnowledgeBaseID: wsTargetKB}) + if err != nil { + t.Fatalf("target: %v", err) + } + t.Cleanup(target.Close) + return target +} + +func (fx *writeSetFixture) backups(t *testing.T, target *ProductionLibraryTarget) *ProductionBackupStore { + t.Helper() + mapping, err := target.Search().MappingSHA256(fx.ctx) + if err != nil { + t.Fatal(err) + } + backups, err := NewProductionBackupStore(ProductionBackupConfig{ + Pool: target.Pool(), Search: target.Search(), Objects: fx.objects, ArchiveRoot: fx.archiveRoot, + Scope: "internal", Identity: target.Identity(), MaxDocuments: 100, + Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, + RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4", + EmbeddingDimension: 1024, IndexMappingSHA256: mapping}, + }) + if err != nil { + t.Fatalf("backup store: %v", err) + } + return backups +} + +// universeDump 是"整实例逐行字节"快照:任何一行被删/被改写都会体现在这里。 +func universeDump(t *testing.T, pool *pgxpool.Pool) map[string][]string { + t.Helper() + dump := map[string][]string{} + for _, table := range productionLibraryTables { + rows, err := pool.Query(context.Background(), fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table)) + if err != nil { + t.Fatalf("dump %s: %v", table, err) + } + values := make([]string, 0, 32) + for rows.Next() { + var raw string + if err := rows.Scan(&raw); err != nil { + rows.Close() + t.Fatalf("dump scan %s: %v", table, err) + } + values = append(values, raw) + } + rows.Close() + if err := rows.Err(); err != nil { + t.Fatalf("dump %s: %v", table, err) + } + dump[table] = values + } + return dump +} + +func diffUniverse(before, after map[string][]string) string { + tables := make([]string, 0, len(before)) + for table := range before { + tables = append(tables, table) + } + sort.Strings(tables) + for _, table := range tables { + if len(before[table]) != len(after[table]) { + return fmt.Sprintf("%s 行数 %d -> %d", table, len(before[table]), len(after[table])) + } + for index := range before[table] { + if before[table][index] != after[table][index] { + return fmt.Sprintf("%s 第 %d 行内容变了:%s", table, index, after[table][index]) + } + } + } + return "" +} + +func requestFor(snapshotID string) libraryrelease.Request { + now := time.Now().UTC() + return libraryrelease.Request{JobID: "job-writeset-acceptance", Target: "internal", SnapshotID: snapshotID, + ParamsDigest: "digest-writeset", Now: now, + Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-writeset", + ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}} +} + +// 场景:目标非空(目标自己的文档/分块/资源/Wiki)、同租户里还有另一个知识库的数据与资源、 +// 一份被两个知识库共享的资源、一个历史孤儿资源、另一个租户的数据;来源快照属于另一个 +// 租户/知识库;资源阶段写失败 → 必须回滚,且"目标原有数据逐行字节不变、本次残留为 0"。 +func TestLibraryRollbackKeepsPreExistingTargetDataIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + before := universeDump(t, fx.pool) + snapshotID := sealSourceSnapshot(t, fx, nil) + target := fx.target(t) + backups := fx.backups(t, target) + + fx.objects.failPath = "/res-src/" + fx.objects.failPut.Store(true) + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil) + fx.objects.failPut.Store(false) + if outcome.State != "rolled_back" { + t.Fatalf("outcome = %+v", outcome) + } + if outcome.FailedAt != libraryrelease.StageResources { + t.Fatalf("failed_at = %s, want resources", outcome.FailedAt) + } + t.Logf("rollback=%s backup=%s ledger=%s", outcome.Rollback, outcome.Backup.SnapshotID, + outcome.Backup.WriteSet.Digest) + after := universeDump(t, fx.pool) + if diff := diffUniverse(before, after); diff != "" { + t.Fatalf("回滚后目标数据被改动:%s", diff) + } + // 历史孤儿不是本次写的:它必须还在(旧实现按"目标租户里没人引用"清理,会删掉它)。 + var orphan string + if err := fx.pool.QueryRow(fx.ctx, `SELECT physical_path FROM resources WHERE id='res-orphan'`).Scan(&orphan); err != nil { + t.Fatalf("历史孤儿资源被删了(恢复把'没人引用的目标租户资源'当成本次残渣):%v", err) + } + var sourceScope int64 + if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2`, + wsSourceTenant, wsSourceKB).Scan(&sourceScope); err != nil { + t.Fatal(err) + } + if sourceScope != 0 { + t.Fatalf("回滚后仍有 %d 行来源范围文档", sourceScope) + } + var residue int64 + if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-src'`).Scan(&residue); err != nil { + t.Fatal(err) + } + if residue != 0 { + t.Fatalf("本次同步新增的资源行未被清理") + } + if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil { + t.Fatalf("回滚必须与预备份一致:%v", err) + } + // 写集合必须已落盘、可校验,并且明确区分"本次新建"与"写前已存在"。 + if outcome.Backup.WriteSet == nil || outcome.Backup.WriteSet.Digest == "" { + t.Fatalf("预备份必须带着已落盘的写集合:%+v", outcome.Backup) + } + ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest) + if err != nil { + t.Fatalf("写集合记录取不回来:%v", err) + } + createdResources := ledger.CreatedIDs("resources") + if len(createdResources) != 1 || createdResources[0] != "res-src" { + t.Fatalf("写集合里本次新建的资源 = %v,want [res-src]", createdResources) + } + // 写集合只允许包含"这次要写"的来源身份;目标自己的资源(含共享/历史孤儿/ + // 其它知识库的资源)一个都不许出现,否则恢复就会按它去删。 + created := map[string]bool{} + for _, id := range ledger.CreatedIDs("resources") { + created[id] = true + } + for _, id := range []string{"res-target", "res-shared", "res-orphan", "res-other", "res-foreign"} { + if created[id] { + t.Fatalf("写集合把目标原有的资源 %s 当成本次新建(恢复会删掉它):%v", id, created) + } + } + if ledger.JobID != "job-writeset-acceptance" || ledger.BackupSnapshotID != outcome.Backup.SnapshotID { + t.Fatalf("写集合没有绑定作业/预备份:%+v", ledger) + } + if ledger.UniverseDigest() == "" { + t.Fatal("写集合必须带整实例内容证据") + } +} + +// 场景:恢复只拿到"快照 id"(worker 重启后的真实形状)→ 没有持久化写集合就不许恢复, +// 更不能把校验降级成"跳过整实例比对"。 +func TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + snapshotID := sealSourceSnapshot(t, fx, nil) + target := fx.target(t) + backups := fx.backups(t, target) + fx.objects.failPath = "/res-src/" + fx.objects.failPut.Store(true) + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil) + fx.objects.failPut.Store(false) + if outcome.State != "rolled_back" { + t.Fatalf("outcome = %+v", outcome) + } + before := universeDump(t, fx.pool) + // 丢掉内存里的写集合:只按快照 id 恢复(这就是 fixedBackupStore 的形状)。 + if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + outcome.Backup.SnapshotID, + SnapshotID: outcome.Backup.SnapshotID}); err == nil { + t.Fatal("缺少持久化写集合记录时恢复必须失败(不许把校验降级成静默跳过)") + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("被拒绝的恢复不许动目标:%s", diff) + } +} + +// 场景:来源快照里的资源 ID 已被另一个租户占用 → 必须在写第一笔之前拒绝, +// 不能 ON CONFLICT 改写别人的行。 +func TestLibraryPreflightRejectsCrossScopeIDConflictIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + // 另一个租户里已经有一条同 ID 的资源行。 + if _, err := fx.pool.Exec(fx.ctx, `INSERT INTO resources VALUES('res-src','`+wsHandleForeign+`',20000,'`+ + wsLocator("backend-foreign", "oss", "foreign-bucket", 20000, "res-src")+`',NULL,'oss',4,'backend-foreign')`); err != nil { + t.Fatal(err) + } + before := universeDump(t, fx.pool) + snapshotID := sealSourceSnapshot(t, fx, nil) + target := fx.target(t) + backups := fx.backups(t, target) + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil) + if outcome.State != "failed" { + t.Fatalf("outcome = %+v", outcome) + } + if outcome.FailedAt != "" { + t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at = %s", outcome.FailedAt) + } + if !strings.Contains(outcome.Error, "冲突") || !strings.Contains(outcome.Error, "res-src") { + t.Fatalf("拒绝原因必须点名冲突的资源 ID:%s", outcome.Error) + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("冲突预检失败后目标被改动:%s", diff) + } + t.Logf("conflict rejected before write: %s", outcome.Error) +} + +// 场景:wiki 阶段半写失败(resources 已成功上传、database 已提交)→ 回滚必须把 +// 本次新增的资源也清掉,目标原有数据逐行不变。 +func TestLibraryWikiStageHalfWriteRollbackIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + before := universeDump(t, fx.pool) + // 同一个作用域里两页同 slug:真实 schema 的唯一索引会让 wiki 阶段写失败, + // 而 database/resources 阶段已经成功提交(半写现场)。 + snapshotID := sealSourceSnapshot(t, fx, func(parts map[string][]map[string]any) { + parts["wiki_pages"] = []map[string]any{ + {"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"}, + {"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"}, + } + }) + target := fx.target(t) + backups := fx.backups(t, target) + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil) + if outcome.State != "rolled_back" { + t.Fatalf("outcome = %+v", outcome) + } + if outcome.FailedAt != libraryrelease.StageWiki { + t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt) + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("wiki 阶段半写回滚后目标数据被改动:%s", diff) + } + if _, _, err := fx.objects.s3.Stat(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src")); err == nil { + t.Log("note: 对象字节保留在桶里(恢复只回滚行,不删对象;对象清理不属于本次范围)") + } + t.Logf("wiki half-write rolled back: %s", outcome.Rollback) +} + +// 场景:写集合记录被删除/篡改 → 恢复必须失败并保持维护,不能"看不见记录就当没写过"。 +func TestLibraryWriteSetRecordTamperRefusesRestoreIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + snapshotID := sealSourceSnapshot(t, fx, nil) + target := fx.target(t) + backups := fx.backups(t, target) + fx.objects.failPath = "/res-src/" + fx.objects.failPut.Store(true) + outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil) + fx.objects.failPut.Store(false) + if outcome.State != "rolled_back" || outcome.Backup.WriteSet == nil { + t.Fatalf("outcome = %+v", outcome) + } + digest := outcome.Backup.WriteSet.Digest + path := writeSetPath(fx.archiveRoot, digest) + + // 1) 记录被篡改:同一个文件改一个字段 → 取回必须失败(摘要对不上)。 + original, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + tampered := bytes.Replace(original, []byte(`"tenant_id":10001`), []byte(`"tenant_id":10002`), 1) + if bytes.Equal(tampered, original) { + t.Fatalf("篡改夹具没生效:%s", original) + } + if err := os.Chmod(path, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, tampered, 0600); err != nil { + t.Fatal(err) + } + if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil { + t.Fatal("写集合被篡改后取回必须失败") + } + // 内存里带着被篡改的记录去恢复,同样必须失败。 + bad := *outcome.Backup.WriteSet + bad.TenantID = 10002 + if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: outcome.Backup.ID, + SnapshotID: outcome.Backup.SnapshotID, WriteSet: &bad}); err == nil { + t.Fatal("写集合内容与摘要不符时恢复必须失败") + } + + // 2) 记录被删除:恢复必须失败并保持维护(不是静默跳过校验)。 + if err := os.WriteFile(path, original, 0600); err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil { + t.Fatal("写集合记录被删除后取回必须失败") + } + t.Logf("tampered/removed ledger refused, digest=%s", digest) +} diff --git a/internal/releaseadapter/v071/library_writeset_kill_test.go b/internal/releaseadapter/v071/library_writeset_kill_test.go new file mode 100644 index 0000000..64688e1 --- /dev/null +++ b/internal/releaseadapter/v071/library_writeset_kill_test.go @@ -0,0 +1,186 @@ +package v071 + +import ( + "context" + "encoding/json" + "io" + "os" + "os/exec" + "strings" + "syscall" + "testing" + "time" + + "knowledge-governance/internal/libraryrelease" +) + +// 真杀进程的接管验证:子进程在 resources 阶段被 SIGKILL(模拟 worker 被杀), +// 已经提交的 database 阶段与"写集合 + 预备份"记录留在盘上;父进程作为"重启后的 +// worker"只按持久记录恢复。 +// +// 说明边界:杀的是**发布路径的进程**(本文件的 helper),不是治理栈里的 worker 容器; +// 队列/UI 侧的接管由父负责页面验收。这里证明的是"进程没了之后,恢复只用盘上的 +// 写集合与整实例证据,并且删不到目标原有数据"。 + +const ( + killChildEnv = "KG_WRITESET_KILL_CHILD" + killStateEnv = "KG_WRITESET_KILL_STATE" + killJournalEnv = "KG_WRITESET_KILL_JOURNAL" +) + +// killHelperState 是子进程写下的"治理库记录"替身:真实系统里这两项分别落在 +// library_jobs.snapshot_id 与 library_versions.evidence.writeset_digest。 +type killHelperState struct { + JobID string `json:"job_id"` + BackupSnapshotID string `json:"backup_snapshot_id"` + TargetSnapshotID string `json:"target_snapshot_id"` + WriteSetDigest string `json:"writeset_digest"` +} + +func TestWriteSetKillHelper(t *testing.T) { + if os.Getenv(killChildEnv) != "1" { + t.Skip("helper process only") + } + // 子进程复用父进程的现场:不重建 schema/索引,写集合与预备份仍落在同一个归档根。 + fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{archiveRoot: os.Getenv(killJournalEnv + "_ARCHIVE"), + index: os.Getenv(killJournalEnv + "_INDEX")}) + state := os.Getenv(killStateEnv) + journal := os.Getenv(killJournalEnv) + snapshotID := os.Getenv("KG_WRITESET_KILL_SNAPSHOT") + target := fx.target(t) + backups := fx.backups(t, target) + current, err := target.Current(fx.ctx) + if err != nil { + t.Fatalf("current: %v", err) + } + backup, err := backups.Prepare(fx.ctx, snapshotID, current) + if err != nil { + t.Fatalf("prepare backup: %v", err) + } + set, err := target.PlanWriteSet(fx.ctx, libraryrelease.WriteSetPlan{JobID: "job-kill", Target: "internal", + BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: snapshotID, Universe: backup.Universe}) + if err != nil { + t.Fatalf("plan write set: %v", err) + } + record, err := json.Marshal(killHelperState{JobID: set.JobID, BackupSnapshotID: backup.SnapshotID, + TargetSnapshotID: snapshotID, WriteSetDigest: set.Digest}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(state, record, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(journal, []byte(fx.archiveRoot), 0600); err != nil { + t.Fatal(err) + } + if err := target.Apply(fx.ctx, libraryrelease.StageDatabase, snapshotID); err != nil { + t.Fatalf("apply database: %v", err) + } + // resources 阶段:第一笔上传就把自己杀掉(真 SIGKILL,没有 defer、没有收尾)。 + dying := &killOnPutStore{inner: fx.objects.s3} + target.config.Objects = dying + if err := target.Apply(fx.ctx, libraryrelease.StageResources, snapshotID); err != nil { + t.Fatalf("apply resources: %v", err) + } + t.Fatal("helper should have been killed during the resources stage") +} + +// killOnPutStore 在第一次上传时直接 SIGKILL 自己:模拟进程被外部杀掉。 +type killOnPutStore struct { + inner *S3ObjectStore +} + +func (s *killOnPutStore) Put(_ context.Context, _, _ string, _ io.Reader, _ int64) error { + if err := syscall.Kill(os.Getpid(), syscall.SIGKILL); err != nil { + return err + } + select {} +} + +func (s *killOnPutStore) Stat(ctx context.Context, provider, path string) (int64, string, error) { + return s.inner.Stat(ctx, provider, path) +} + +func (s *killOnPutStore) Copy(ctx context.Context, provider, source, target string) error { + return s.inner.Copy(ctx, provider, source, target) +} + +func TestLibraryKillWorkerThenResumeRecoversWriteSetIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + before := universeDump(t, fx.pool) + snapshotID := sealSourceSnapshot(t, fx, nil) + + statePath := t.TempDir() + "/kill.json" + journalPath := t.TempDir() + "/journal.txt" + command := exec.Command(os.Args[0], "-test.run=TestWriteSetKillHelper", "-test.v") + command.Env = append(os.Environ(), killChildEnv+"=1", killStateEnv+"="+statePath, + killJournalEnv+"="+journalPath, killJournalEnv+"_ARCHIVE="+fx.archiveRoot, + killJournalEnv+"_INDEX="+fx.index, "KG_WRITESET_KILL_SNAPSHOT="+snapshotID) + output, err := command.CombinedOutput() + if err == nil { + t.Fatalf("helper must be killed, it exited cleanly: %s", output) + } + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("helper failed to run: %v\n%s", err, output) + } + status, ok := exitErr.Sys().(syscall.WaitStatus) + if !ok || !status.Signaled() || status.Signal() != syscall.SIGKILL { + t.Fatalf("helper must die from SIGKILL, got %v\n%s", err, output) + } + t.Logf("helper killed by %s (half-written target)", status.Signal()) + + raw, err := os.ReadFile(statePath) + if err != nil { + t.Fatalf("child did not persist the job record: %v", err) + } + var record killHelperState + if err := json.Unmarshal(raw, &record); err != nil { + t.Fatal(err) + } + if record.WriteSetDigest == "" || record.BackupSnapshotID == "" { + t.Fatalf("child must persist writeset digest + backup snapshot: %+v", record) + } + journalRoot, err := os.ReadFile(journalPath) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(string(journalRoot)) != fx.archiveRoot { + t.Fatalf("journal root mismatch: %q", journalRoot) + } + + // 重启后的 worker:新连接、新目标对象,只拿盘上的记录。 + resumedTarget := fx.target(t) + ledger, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest) + if err != nil { + t.Fatalf("重启后取回写集合失败:%v", err) + } + if ledger.JobID != record.JobID || ledger.BackupSnapshotID != record.BackupSnapshotID || + ledger.TargetSnapshotID != record.TargetSnapshotID { + t.Fatalf("写集合与作业记录不一致:%+v vs %+v", ledger, record) + } + if err := resumedTarget.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + record.BackupSnapshotID, + SnapshotID: record.BackupSnapshotID, WriteSet: ledger}); err != nil { + t.Fatalf("接管恢复失败:%v", err) + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("接管恢复后目标数据被改动:%s", diff) + } + if err := resumedTarget.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: record.BackupSnapshotID}); err != nil { + t.Fatalf("接管恢复后必须与预备份一致:%v", err) + } + t.Logf("resumed after SIGKILL: job=%s backup=%s ledger=%s", record.JobID, record.BackupSnapshotID, record.WriteSetDigest) + + // 负例:记录被移除 → 恢复必须失败(维护保持),而不是"看不见就当没写过"。 + if err := os.Chmod(writeSetPath(fx.archiveRoot, record.WriteSetDigest), 0600); err != nil { + t.Fatal(err) + } + if err := os.Remove(writeSetPath(fx.archiveRoot, record.WriteSetDigest)); err != nil { + t.Fatal(err) + } + if _, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest); err == nil { + t.Fatal("写集合被移除后必须取不回来") + } + _ = time.Now() +} diff --git a/scripts/test-scope.json b/scripts/test-scope.json index 80e0103..bb386fc 100644 --- a/scripts/test-scope.json +++ b/scripts/test-scope.json @@ -187,6 +187,41 @@ "package": "knowledge-governance/internal/qawemail", "test": "TestScheduledSendRequiresConfirmationAndSendsExactlyOnce", "reason": "需要一个一次性 PostgreSQL 与测试主密钥(KG_WEEKLY_MAIL_TEST_DSN/KG_WEEKLY_MAIL_TEST_KEY),CI 不提供;本批已在隔离库真实跑通并通过" + }, + { + "package": "knowledge-governance/internal/releaseadapter/v071", + "test": "TestLibraryRollbackKeepsPreExistingTargetDataIsolated", + "reason": "需要一次性隔离库 kg_library_test_*、随机 ES 索引与真实对象存储(KG_LIBRARY_TEST_S3_*);验证目标非空/同租户其它知识库/共享资源/历史孤儿/跨租户数据在失败恢复后逐行字节不变,默认 CI 无这些服务,本机已实跑通过" + }, + { + "package": "knowledge-governance/internal/releaseadapter/v071", + "test": "TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated", + "reason": "同上环境;验证缺少持久化写集合时恢复必须失败且不动目标,默认 CI 跳过不算验证" + }, + { + "package": "knowledge-governance/internal/releaseadapter/v071", + "test": "TestLibraryPreflightRejectsCrossScopeIDConflictIsolated", + "reason": "同上环境;验证跨范围同 ID 资源冲突在任何写入之前被拒绝,默认 CI 跳过不算验证" + }, + { + "package": "knowledge-governance/internal/releaseadapter/v071", + "test": "TestLibraryWikiStageHalfWriteRollbackIsolated", + "reason": "同上环境;验证 wiki 阶段半写(database/resources 已提交)后回滚不留残渣,默认 CI 跳过不算验证" + }, + { + "package": "knowledge-governance/internal/releaseadapter/v071", + "test": "TestLibraryWriteSetRecordTamperRefusesRestoreIsolated", + "reason": "同上环境;验证写集合记录被篡改/删除后取回与恢复都失败,默认 CI 跳过不算验证" + }, + { + "package": "knowledge-governance/internal/releaseadapter/v071", + "test": "TestLibraryKillWorkerThenResumeRecoversWriteSetIsolated", + "reason": "同上环境并用 SIGKILL 杀掉子进程模拟 worker 被杀,验证重启后只按持久写集合恢复且目标原有数据不变;需要本地真实 PG/ES/MinIO,默认 CI 跳过不算验证" + }, + { + "package": "knowledge-governance/internal/releaseadapter/v071", + "test": "TestWriteSetKillHelper", + "reason": "上面 SIGKILL 用例的子进程 fixture,单独运行没有意义;默认 CI 跳过" } ] }