fix(library): 写集合台账与跨范围冲突预检
build / build (push) Successful in 1m34s

失败恢复会在写完之后按"目标租户里没人引用"清理资源行,于是历史孤儿、同租户其它
知识库暂未绑定的资源、被共享的资源都可能被删掉;而预备份里没有这些行,删了不可
恢复,随后整实例指纹才发现不一致,只能停在 rollback_failed。恢复校验依赖的整实例
证据又只存在内存里,worker 重启后只按快照 id 恢复,等于静默跳过比对。

改法:写第一笔之前把"本次明确写集合 + 写前存在状态 + 预备份时刻整实例内容证据"
算出来并按内容寻址落盘(文件名与治理库里的引用都是 sha256),摘要登记在预备份
版本证据里;恢复只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复内容里"的精确 ID,并对
资源额外要求仍无绑定/正文引用;缺记录、记录被改、绑定对不上都直接失败、维护保持。
resources 的 upsert 补租户限定,跨范围同 ID 在写入之前就被预检拒绝。
整实例指纹从"身份集合"改成"逐行内容摘要",别的知识库同 ID 行被覆盖写坏也会暴露。

本地验证(已跑):
- make verify 15 项全 success(identity 因本地档案 skipped)
- go test ./internal/releaseadapter/v071/ -run Isolated(真实 PG + 真实 ES +
  真实 MinIO 对象存储,写失败按定位符注入):目标非空/同租户其它知识库/共享资源/
  历史孤儿/跨租户数据在失败恢复后逐行字节不变、本次残留 0;缺写集合拒绝恢复;
  跨范围 ID 冲突在写入前拒绝;wiki 阶段半写回滚干净
- SIGKILL 子进程后按盘上写集合接管恢复,目标原有数据逐行不变
- go test ./internal/libraryrelease/:写集合先于第一笔、规划失败/登记失败都不写目标、
  接管不重新规划
This commit is contained in:
姚章浩
2026-10-29 23:55:13 +08:00
parent 034c0b3a1c
commit e0768f2caa
16 changed files with 2103 additions and 99 deletions
+16 -1
View File
@@ -28,6 +28,7 @@ import (
"knowledge-governance/internal/libraryfreeze"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarystack"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
type drillVerification struct {
@@ -220,7 +221,21 @@ WHERE id = (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$
log.Printf("drill: drift detected as expected (%s)", drift.Error)
// 4) 按预备份恢复。
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID}); err != nil {
// 恢复必须带着写集合记录。演练只改了一行既有内容(canary),没有新建行,
// 所以写集合为空;整实例证据用预备份时刻的那一份。
identityValue := target.Identity()
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
if err := ledger.Seal(); err != nil {
return "", errors.New("演练写集合生成失败:" + err.Error())
}
if _, err := v071.PersistWriteSet(config.LibraryArchiveRoot, &ledger); err != nil {
return "", errors.New("演练写集合落盘失败:" + err.Error())
}
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
WriteSet: &ledger}); err != nil {
return "", errors.New("按预备份恢复失败:" + err.Error())
}
// 5) 恢复后逐类复核:必须与预备份完全一致。
+16 -10
View File
@@ -291,12 +291,24 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
var backupSource libraryrelease.BackupStore = backups
if resumed {
backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
SnapshotID: previous.SnapshotID}}
// 接管必须带着原来那份写集合:没有它,恢复只能按目标现状推断残渣,
// 而那时目标已经被写过(真实缺陷:历史孤儿资源被当成残渣删掉)。
set, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
payload.LibraryJobID, previous.SnapshotID)
if ledgerErr != nil {
// 读不到就一笔都不许写:目标可能半写,恢复口径未知。
message := "接管时读不回写集合记录,拒绝继续写目标:" + ledgerErr.Error()
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
recorder.Stage(ctx, "prebackup", "failed", message)
return errors.New(message)
}
backupSource = ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
SnapshotID: previous.SnapshotID}, write: set}
}
request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID,
ParamsDigest: digest, Now: now,
request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope,
SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
@@ -506,12 +518,6 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Stor
}
// fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。
type fixedBackupStore struct{ ref libraryrelease.BackupRef }
func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
return s.ref, nil
}
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误:
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
+20 -2
View File
@@ -64,6 +64,16 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
}
prebackup := windowJob.SnapshotID
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
payload.WindowJobID, prebackup)
if ledgerErr != nil {
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
return errors.New(note)
}
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
if wireErr != nil {
@@ -78,7 +88,7 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config
defer control.Close()
defer coordinator.Close()
restorer := recoveryRestorer{target: target, expected: prebackup}
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger}
verifier := recoveryVerifier{target: target, expected: prebackup}
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
now := time.Now().UTC()
@@ -106,13 +116,21 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config
type recoveryRestorer struct {
target *v071.ProductionLibraryTarget
expected string
writeSet *libraryrelease.WriteSet
}
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
if backupID != r.expected {
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
}
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID})
if r.writeSet == nil {
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
}
if r.writeSet.BackupSnapshotID != backupID {
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
}
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID,
WriteSet: r.writeSet})
}
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
+140
View File
@@ -0,0 +1,140 @@
package main
import (
"context"
"errors"
"fmt"
"sort"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// 写集合台账在治理侧的锚点:摘要登记在"这次作业的预备份版本"证据里。
// 为什么不用新表:版本记录本来就绑定 scope + snapshot + 作业,且 id 由这三者
// 决定;把摘要挂上去,重启后按同一身份就能取回,不需要改 schema。
// writeSetDigestOf 读某次作业/某份预备份登记的写集合摘要。
// 读不到就返回错误:缺失必须让调用方失败,不能降级成"没有记录就当没写过"。
func writeSetDigestOf(ctx context.Context, store *postgresstore.Store, scope, jobID, snapshotID string) (string, error) {
versions, err := store.LibraryVersions(ctx, scope, 200)
if err != nil {
return "", fmt.Errorf("读不到版本记录:%w", err)
}
for _, version := range versions {
if version.Kind != governance.LibraryVersionProductionPrebackup || version.SnapshotID != snapshotID {
continue
}
if version.SourceJobID != jobID {
continue
}
raw, _ := version.Evidence["writeset_digest"].(string)
if raw == "" {
return "", fmt.Errorf("预备份 %s 没有登记写集合摘要(作业 %s)", snapshotID, jobID)
}
return raw, nil
}
return "", fmt.Errorf("找不到作业 %s 的预备份版本记录(快照 %s)", jobID, snapshotID)
}
// loadWriteSetForJob 按"作业 + 目标 + 预备份"取回写集合并逐项核对绑定。
// 任何一步不符都返回错误:恢复宁可保持维护,也不许用一份对不上的记录去删目标数据。
func loadWriteSetForJob(ctx context.Context, store *postgresstore.Store, archiveRoot, scope, jobID, snapshotID string) (*libraryrelease.WriteSet, error) {
digest, err := writeSetDigestOf(ctx, store, scope, jobID, snapshotID)
if err != nil {
return nil, err
}
set, err := v071.LoadWriteSet(archiveRoot, digest)
if err != nil {
return nil, err
}
if set.JobID != jobID {
return nil, fmt.Errorf("写集合记录属于作业 %s,不是 %s", set.JobID, jobID)
}
if set.Target != scope {
return nil, fmt.Errorf("写集合记录属于目标 %s,不是 %s", set.Target, scope)
}
if set.BackupSnapshotID != snapshotID {
return nil, fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", set.BackupSnapshotID, snapshotID)
}
return set, nil
}
// WriteSetPlanned 把写集合摘要登记进"这次作业的预备份版本"证据。
// 它在第一笔写入之前被调用;失败即中止(调用方据此不写目标)。
func (r libraryStageRecorder) WriteSetPlanned(ctx context.Context, set libraryrelease.WriteSet) error {
versions, err := r.store.LibraryVersions(ctx, r.scope, 200)
if err != nil {
return fmt.Errorf("读不到现有版本证据:%w", err)
}
var existing map[string]any
for _, version := range versions {
if version.Kind == governance.LibraryVersionProductionPrebackup && version.SnapshotID == set.BackupSnapshotID {
existing = version.Evidence
break
}
}
evidence := map[string]any{}
for key, value := range existing {
evidence[key] = value
}
// 保留预备份原有证据(备份字节、索引映射等),只追加写集合锚点。
evidence["backup_snapshot"] = set.BackupSnapshotID
evidence["job"] = set.JobID
if r.mappingDigest != "" {
evidence["index_mapping_sha256"] = r.mappingDigest
}
evidence["writeset_digest"] = set.Digest
evidence["writeset_universe_digest"] = set.UniverseDigest()
evidence["writeset_target_snapshot"] = set.TargetSnapshotID
evidence["writeset_rows"] = len(set.Rows)
return r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, set.BackupSnapshotID),
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: set.BackupSnapshotID,
SourceJobID: r.jobID, CreatedBy: "worker", Evidence: evidence, CreatedAt: time.Now().UTC(),
})
}
// ledgerBackupStore 是"接管/恢复"用的预备份来源:只认已经持久化的那一份,
// 并把它绑定的写集合一起带出来。
//
// 为什么不能像以前那样只返回 snapshot id:那一刻目标已经被写过,恢复若拿不到
// 写集合就只能按目标现状推断残渣——那正是会误删历史资源的路径。所以这里取不到
// 记录就直接失败,而不是返回一个"没有 Universe 的引用"。
type ledgerBackupStore struct {
ref libraryrelease.BackupRef
write *libraryrelease.WriteSet
}
func (s ledgerBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
if s.write == nil {
return libraryrelease.BackupRef{}, errors.New("恢复缺少持久化的写集合记录:拒绝在没有可追溯写集合的情况下继续")
}
ref := s.ref
ref.WriteSet = s.write
return ref, nil
}
// sortedWriteSetSummary 只用于日志/证据:把本次写集合按表汇总成稳定的短摘要。
func sortedWriteSetSummary(set libraryrelease.WriteSet) string {
counts := map[string]int{}
for _, row := range set.Rows {
counts[row.Table]++
}
tables := make([]string, 0, len(counts))
for table := range counts {
tables = append(tables, table)
}
sort.Strings(tables)
summary := ""
for _, table := range tables {
if summary != "" {
summary += ","
}
summary += fmt.Sprintf("%s=%d", table, counts[table])
}
return summary
}
+33 -2
View File
@@ -36,6 +36,9 @@ type Grant struct {
// Request 是一次待执行的生产同步。
type Request struct {
// JobID 是这次发布的作业身份:写集合记录与预备份引用都绑定它,
// worker 重启后凭它找回"这次写了什么"。
JobID string
Target string
SnapshotID string
// ParamsDigest 覆盖会影响生产结果的全部参数(目标、快照、排除范围等)。
@@ -75,9 +78,11 @@ type BackupRef struct {
SnapshotID string `json:"snapshot_id"`
Bytes int64 `json:"bytes"`
CreatedAt time.Time `json:"created_at"`
// Universe 是预备份时刻的整实例指纹,恢复后逐表比对。空表示这次没有采集
// (例如 worker 被杀后用已持久化的快照接管),此时恢复只能靠内容校验兜底。
// Universe 是预备份时刻的整实例指纹,恢复后逐表比对。
Universe map[string]TableFingerprint `json:"universe,omitempty"`
// WriteSet 是本次发布的写集合与整实例内容证据。它在写第一笔之前就落盘并
// 登记摘要;恢复必须按它清理,缺了它恢复要失败(不许降级成"跳过比对")。
WriteSet *WriteSet `json:"write_set,omitempty"`
}
type Target interface {
@@ -203,6 +208,32 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor
}
recorder.Stage(ctx, "prebackup", "succeeded", "预备份 "+backup.ID)
// 写集合必须在第一笔写入之前落盘并登记:恢复只按"这次明确写了什么"清理,
// 不按目标现状推断残渣(那会把历史孤儿/共享资源当成本次残留删掉)。
if backup.WriteSet == nil {
if planner, ok := target.(WriteSetPlanner); ok {
set, planErr := planner.PlanWriteSet(ctx, WriteSetPlan{JobID: request.JobID, Target: request.Target,
BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: request.SnapshotID, Universe: backup.Universe})
if planErr != nil {
outcome.State = "failed"
outcome.Error = "写集合/冲突预检失败,未对生产执行任何写入:" + planErr.Error()
recorder.Stage(ctx, "prebackup", "failed", outcome.Error)
return outcome
}
backup.WriteSet = &set
outcome.Backup = backup
if observer, ok := recorder.(WriteSetObserver); ok {
if err := observer.WriteSetPlanned(ctx, set); err != nil {
// 硬门禁:记录没落库 = 恢复时不知道写集合 = 一笔都不许写。
outcome.State = "failed"
outcome.Error = "写集合已落盘但引用无法持久化,未对生产执行任何写入:" + err.Error()
recorder.Stage(ctx, "prebackup", "failed", outcome.Error)
return outcome
}
}
}
}
for _, stage := range Stages() {
recorder.Stage(ctx, string(stage), "running", "")
if err = target.Apply(ctx, stage, request.SnapshotID); err != nil {
+159
View File
@@ -250,3 +250,162 @@ func TestRunReportsRollbackFailureDistinctly(t *testing.T) {
t.Fatalf("stage log = %v", recorder.entries)
}
}
// writeSetPlanTarget 记录"写集合规划"与阶段写入的相对顺序:写集合必须在第一笔之前。
type writeSetPlanTarget struct {
*fakeTarget
planErr error
plans []WriteSetPlan
events *[]string
ledger *WriteSet
restores []BackupRef
}
func (p *writeSetPlanTarget) PlanWriteSet(_ context.Context, plan WriteSetPlan) (WriteSet, error) {
*p.events = append(*p.events, "plan")
p.plans = append(p.plans, plan)
if p.planErr != nil {
return WriteSet{}, p.planErr
}
set := WriteSet{Version: WriteSetVersion, JobID: plan.JobID, Target: plan.Target,
TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: plan.BackupSnapshotID,
TargetSnapshotID: plan.TargetSnapshotID, CreatedAt: time.Now().UTC(), Universe: plan.Universe,
Rows: []WriteSetRow{{Table: "resources", ID: "res-new"}}}
if err := set.Seal(); err != nil {
return WriteSet{}, err
}
p.ledger = &set
return set, nil
}
func (p *writeSetPlanTarget) Apply(ctx context.Context, stage Stage, snapshot string) error {
*p.events = append(*p.events, "apply:"+string(stage))
return p.fakeTarget.Apply(ctx, stage, snapshot)
}
func (p *writeSetPlanTarget) Restore(ctx context.Context, backup BackupRef) error {
p.restores = append(p.restores, backup)
return p.fakeTarget.Restore(ctx, backup)
}
type writeSetRecorder struct {
*stageLog
events *[]string
err error
sets []WriteSet
}
func (r *writeSetRecorder) WriteSetPlanned(_ context.Context, set WriteSet) error {
*r.events = append(*r.events, "observe")
r.sets = append(r.sets, set)
return r.err
}
func plannedRun(t *testing.T) (*writeSetPlanTarget, *writeSetRecorder, *[]string, Request, *fakeBackups) {
t.Helper()
now := time.Now()
events := []string{}
target := &writeSetPlanTarget{fakeTarget: &fakeTarget{}, events: &events}
recorder := &writeSetRecorder{stageLog: &stageLog{}, events: &events}
backups := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap", Bytes: 512,
CreatedAt: now.UTC(), Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}}
request := request(now)
return target, recorder, &events, request, backups
}
func TestRunPlansWriteSetBeforeFirstWrite(t *testing.T) {
target, recorder, events, request, backups := plannedRun(t)
outcome := Run(context.Background(), request, target, backups, recorder)
if outcome.State != "succeeded" {
t.Fatalf("outcome = %+v", outcome)
}
if len(target.plans) != 1 {
t.Fatalf("write set must be planned exactly once, got %d", len(target.plans))
}
if got := (*events)[0]; got != "plan" {
t.Fatalf("write set must be planned before anything touches the target, first event = %s", got)
}
if len(recorder.sets) != 1 || recorder.sets[0].Digest == "" {
t.Fatalf("planner output must be persisted before writing: %+v", recorder.sets)
}
if target.plans[0].JobID != request.JobID || target.plans[0].TargetSnapshotID != request.SnapshotID ||
target.plans[0].Target != request.Target {
t.Fatalf("plan must bind job/target/snapshot: %+v", target.plans[0])
}
if target.plans[0].BackupSnapshotID != "backup-snap" {
t.Fatalf("plan must bind the backup snapshot: %+v", target.plans[0])
}
if len(*events) < 2 || (*events)[1] != "observe" {
t.Fatalf("write set must be persisted right after planning, events=%v", *events)
}
}
func TestRunStopsBeforeWritingWhenWriteSetPlanFails(t *testing.T) {
target, recorder, events, request, backups := plannedRun(t)
target.planErr = errors.New("资源 ID 冲突:res-x 已属于其它知识库")
outcome := Run(context.Background(), request, target, backups, recorder)
if outcome.State != "failed" {
t.Fatalf("outcome = %+v", outcome)
}
if !strings.Contains(outcome.Error, "冲突") {
t.Fatalf("error must carry the conflict reason: %s", outcome.Error)
}
if len(target.applied) != 0 {
t.Fatalf("no stage may run after a failed preflight: %v", target.applied)
}
if len(recorder.sets) != 0 {
t.Fatal("a failed plan must not be persisted as the write set")
}
for _, event := range *events {
if strings.HasPrefix(event, "apply:") {
t.Fatalf("target was written despite a failed preflight: %v", *events)
}
}
}
func TestRunStopsBeforeWritingWhenWriteSetCannotBePersisted(t *testing.T) {
target, recorder, events, request, backups := plannedRun(t)
recorder.err = errors.New("治理库写不进去")
outcome := Run(context.Background(), request, target, backups, recorder)
if outcome.State != "failed" {
t.Fatalf("outcome = %+v", outcome)
}
if !strings.Contains(outcome.Error, "写集合") {
t.Fatalf("error must name the write-set record: %s", outcome.Error)
}
if len(target.applied) != 0 {
t.Fatalf("no stage may run when the write set cannot be persisted: %v", target.applied)
}
for _, event := range *events {
if strings.HasPrefix(event, "apply:") {
t.Fatalf("target was written without a persisted write set: %v", *events)
}
}
}
// 接管时必须复用持久化的写集合,而不是按当前(已半写的)目标重新规划。
func TestRunResumeUsesPersistedWriteSet(t *testing.T) {
target, recorder, events, request, backups := plannedRun(t)
resumed := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap",
WriteSet: &WriteSet{Version: WriteSetVersion, JobID: request.JobID, Target: request.Target,
TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: "backup-snap",
TargetSnapshotID: request.SnapshotID, CreatedAt: time.Now().UTC(),
Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}}}
backups = resumed
target.fakeTarget.applyErr = map[Stage]error{StageResources: errors.New("上传失败")}
outcome := Run(context.Background(), request, target, backups, recorder)
if outcome.State != "rolled_back" && outcome.State != "rollback_failed" {
t.Fatalf("outcome = %+v", outcome)
}
if len(target.plans) != 0 {
t.Fatalf("resume must not re-plan the write set: %+v", target.plans)
}
if len(target.restores) != 1 || target.restores[0].WriteSet == nil {
t.Fatalf("restore must carry the persisted write set: %+v", target.restores)
}
for _, event := range *events {
if event == "plan" {
t.Fatalf("resume re-planned against a half-written target: %v", *events)
}
}
}
+200
View File
@@ -0,0 +1,200 @@
package libraryrelease
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"time"
)
// WriteSetVersion 是写集合台账的格式版本。读到的版本不认识就拒绝恢复:
// 宁可保持维护,也不要用一份"格式变了但看起来还能读"的记录去删目标数据。
const WriteSetVersion = 1
// WriteSetRow 是台账里的一行:本次发布明确要写的精确身份 + 写前状态。
//
// 为什么需要"写前状态":恢复不能在目标里按"没人引用"推断残渣。历史孤儿资源、
// 同租户另一个知识库暂未绑定的资源、被多个知识库共享的资源,都满足"没人引用",
// 但它们不是这次作业写的,删掉就是数据丢失(快照里没有这些行,恢复不回来)。
// 只有写前不存在、且由本次写进去的行才允许清理。
type WriteSetRow struct {
Table string `json:"table"`
ID string `json:"id"`
// Existing 表示写前目标里已经有这一行(属于目标自己的数据,恢复一律不许删)。
Existing bool `json:"existing"`
// Owner 是写前该行的归属("<tenant>/<knowledge_base>"),不存在时为空。
Owner string `json:"owner,omitempty"`
}
// WriteSetPlan 是调用方在写第一笔之前交给目标的信息:绑定作业、目标、预备份与快照,
// 以及预备份时刻的整实例证据。
type WriteSetPlan struct {
JobID string
Target string
BackupSnapshotID string
TargetSnapshotID string
Universe map[string]TableFingerprint
}
// WriteSet 是"这一次发布的写集合 + 写前整实例内容证据",内容寻址、可校验、
// 与 job/target/backup 绑定。恢复只读它,不读目标现状去推断。
type WriteSet struct {
Version int `json:"version"`
JobID string `json:"job_id"`
Target string `json:"target"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
BackupSnapshotID string `json:"backup_snapshot_id"`
TargetSnapshotID string `json:"target_snapshot_id"`
CreatedAt time.Time `json:"created_at"`
Rows []WriteSetRow `json:"rows"`
Universe map[string]TableFingerprint `json:"universe"`
// Digest 是对除本字段外全部内容的 sha256:落盘文件名与治理库里的引用都用它,
// 记录被改一个字节就对不上。
Digest string `json:"digest"`
}
// Seal 把行序规范化为可复现的顺序并写入摘要。调用方在落盘与登记之前必须先 Seal。
// 排序在这里落地(而不是只在算摘要时排),否则落盘字节与摘要覆盖的内容不一致。
func (w *WriteSet) Seal() error {
sort.Slice(w.Rows, func(i, j int) bool {
if w.Rows[i].Table != w.Rows[j].Table {
return w.Rows[i].Table < w.Rows[j].Table
}
return w.Rows[i].ID < w.Rows[j].ID
})
digest, err := w.computeDigest()
if err != nil {
return err
}
w.Digest = digest
return nil
}
// CanonicalBytes 是不含摘要字段的规范字节:摘要覆盖的就是它,落盘文件另外
// 带上摘要本身(便于自描述),读回时用同一份规范字节重新推导。
func (w WriteSet) CanonicalBytes() ([]byte, error) {
copied := w
copied.Digest = ""
rows := make([]WriteSetRow, len(w.Rows))
copy(rows, w.Rows)
// 行序不影响语义但影响摘要:两边都按 (table,id) 排序才能算出同一个值。
sort.Slice(rows, func(i, j int) bool {
if rows[i].Table != rows[j].Table {
return rows[i].Table < rows[j].Table
}
return rows[i].ID < rows[j].ID
})
copied.Rows = rows
raw, err := json.Marshal(copied)
if err != nil {
return nil, fmt.Errorf("编码写集合失败: %w", err)
}
return raw, nil
}
func (w *WriteSet) computeDigest() (string, error) {
raw, err := w.CanonicalBytes()
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
// Verify 校验摘要与必备身份。缺任一身份都必须失败:没有绑定就没有"这份记录属于
// 这个作业/目标/预备份"的证明,拿它去删目标数据无法追责。
func (w *WriteSet) Verify() error {
if w.Version != WriteSetVersion {
return fmt.Errorf("写集合记录版本 %d 不受支持(当前 %d)", w.Version, WriteSetVersion)
}
if w.JobID == "" || w.BackupSnapshotID == "" || w.TargetSnapshotID == "" {
return errors.New("写集合记录缺少作业/预备份/快照绑定")
}
if w.TenantID == 0 || w.KnowledgeBaseID == "" {
return errors.New("写集合记录缺少目标范围身份")
}
if len(w.Universe) == 0 {
return errors.New("写集合记录缺少整实例内容证据")
}
if w.Digest == "" {
return errors.New("写集合记录没有摘要")
}
digest, err := w.computeDigest()
if err != nil {
return err
}
if digest != w.Digest {
return errors.New("写集合记录摘要不符(内容被改动或截断)")
}
return nil
}
// CreatedIDs 返回"本次新建"的精确身份(写前不存在的行)。恢复只允许删这些。
func (w WriteSet) CreatedIDs(table string) []string {
ids := make([]string, 0, 8)
for _, row := range w.Rows {
if row.Table == table && !row.Existing {
ids = append(ids, row.ID)
}
}
sort.Strings(ids)
return ids
}
// ExistingIDs 返回写前已存在的身份(用于恢复后复核"目标自己的行都还在")。
func (w WriteSet) ExistingIDs(table string) []string {
ids := make([]string, 0, 8)
for _, row := range w.Rows {
if row.Table == table && row.Existing {
ids = append(ids, row.ID)
}
}
sort.Strings(ids)
return ids
}
// UniverseDigest 是整实例证据的摘要,随台账一起登记,便于独立核对。
func (w WriteSet) UniverseDigest() string {
if len(w.Universe) == 0 {
return ""
}
tables := make([]string, 0, len(w.Universe))
for table := range w.Universe {
tables = append(tables, table)
}
sort.Strings(tables)
type pair struct {
Count int64 `json:"count"`
Digest string `json:"digest"`
}
ordered := make([]pair, 0, len(tables))
for _, table := range tables {
ordered = append(ordered, pair{Count: w.Universe[table].Count, Digest: w.Universe[table].Digest})
}
raw, err := json.Marshal(struct {
Tables []string `json:"tables"`
Values []pair `json:"values"`
}{Tables: tables, Values: ordered})
if err != nil {
return ""
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])
}
// WriteSetPlanner 由能访问目标与归档的实现提供:它在写第一笔之前算出写集合、
// 拒绝跨范围 ID 冲突,并把记录落盘。失败即"一笔都不许写"。
type WriteSetPlanner interface {
PlanWriteSet(ctx context.Context, plan WriteSetPlan) (WriteSet, error)
}
// WriteSetObserver 让调用方在写第一笔之前把写集合摘要持久化到治理库。
// 失败同样意味着"一笔都不许写":没有可追溯的写集合,恢复就只能靠推断。
type WriteSetObserver interface {
WriteSetPlanned(ctx context.Context, set WriteSet) error
}
+104
View File
@@ -0,0 +1,104 @@
package libraryrelease
import (
"encoding/json"
"strings"
"testing"
"time"
)
func sampleWriteSet() WriteSet {
return WriteSet{Version: WriteSetVersion, JobID: "job-1", Target: "internal", TenantID: 10001,
KnowledgeBaseID: "kb", BackupSnapshotID: "backup-1", TargetSnapshotID: "snap-1",
CreatedAt: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC),
Rows: []WriteSetRow{
{Table: "resources", ID: "res-new"},
{Table: "knowledges", ID: "doc-existing", Existing: true, Owner: "10001/kb"},
{Table: "resources", ID: "res-old", Existing: true, Owner: "10001/"},
},
Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "abc"}}}
}
func TestWriteSetSealVerifyRoundTrip(t *testing.T) {
set := sampleWriteSet()
if err := set.Seal(); err != nil {
t.Fatal(err)
}
if set.Digest == "" {
t.Fatal("seal must produce a digest")
}
if err := set.Verify(); err != nil {
t.Fatalf("sealed record must verify: %v", err)
}
// 行序不影响摘要:换序后重新 Sealed 必须得到同一个值(落盘/读回才可比)。
shuffled := sampleWriteSet()
shuffled.Rows[0], shuffled.Rows[2] = shuffled.Rows[2], shuffled.Rows[0]
if err := shuffled.Seal(); err != nil {
t.Fatal(err)
}
if shuffled.Digest != set.Digest {
t.Fatalf("digest must not depend on row order: %s vs %s", shuffled.Digest, set.Digest)
}
// 序列化往返(落盘再读回)必须仍然通过校验。
raw, err := json.Marshal(shuffled)
if err != nil {
t.Fatal(err)
}
var reloaded WriteSet
if err := json.Unmarshal(raw, &reloaded); err != nil {
t.Fatal(err)
}
if err := reloaded.Verify(); err != nil {
t.Fatalf("record read back from disk must verify: %v", err)
}
}
func TestWriteSetVerifyRejectsTamperingAndMissingEvidence(t *testing.T) {
cases := map[string]func(*WriteSet){
"内容被改": func(set *WriteSet) {
set.Rows[0].ID = "res-other"
},
"换租户": func(set *WriteSet) {
set.TenantID = 20000
},
"整实例证据被清空": func(set *WriteSet) {
set.Universe = nil
},
"缺预备份绑定": func(set *WriteSet) {
set.BackupSnapshotID = ""
},
"缺摘要": func(set *WriteSet) {
set.Digest = ""
},
"版本不认识": func(set *WriteSet) {
set.Version = 99
},
}
for name, mutate := range cases {
set := sampleWriteSet()
if err := set.Seal(); err != nil {
t.Fatal(err)
}
mutate(&set)
if err := set.Verify(); err == nil {
t.Fatalf("%s:必须校验失败", name)
}
}
}
func TestWriteSetCreatedIDsOnlyCoversRowsCreatedByThisRelease(t *testing.T) {
set := sampleWriteSet()
if err := set.Seal(); err != nil {
t.Fatal(err)
}
created := set.CreatedIDs("resources")
if len(created) != 1 || created[0] != "res-new" {
t.Fatalf("created resources = %v, want [res-new](写前已存在的资源绝不能被当成残渣)", created)
}
if strings.Join(set.ExistingIDs("resources"), ",") != "res-old" {
t.Fatalf("existing resources = %v", set.ExistingIDs("resources"))
}
if set.UniverseDigest() == "" {
t.Fatal("universe digest must be derived from the whole-instance evidence")
}
}
@@ -28,6 +28,11 @@ func TestLibraryExportIsolatedPostgres(t *testing.T) {
t.Fatal(err)
}
defer db.Close()
// 同一个隔离库会被本包的多个用例复用:不清掉上一次的 schema 就会
// "relation already exists",让人误以为是导出本身坏了。
if _, err = db.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
t.Fatal(err)
}
if _, err = db.Exec(ctx, libraryTestSchema); err != nil {
t.Fatal(err)
}
@@ -395,9 +395,11 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc
}
sort.Strings(columns)
// 本库行先删后插:除 resources(可能被多个知识库共享)外不需要 upsert。
// resources 的 upsert 必须限定"同租户":同 ID 属于别的租户时整行覆盖会把
// 别人的文件改坏(跨范围 ID 冲突已由写集合预检拒绝,这里是最后一道闸)。
if table == "resources" {
statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb)
ON CONFLICT (id) DO UPDATE SET %s`, table, table, updateAssignments(columns))
ON CONFLICT (id) DO UPDATE SET %s WHERE %s.tenant_id = $2`, table, table, updateAssignments(columns), table)
} else {
statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb)`,
table, table)
@@ -407,9 +409,19 @@ ON CONFLICT (id) DO UPDATE SET %s`, table, table, updateAssignments(columns))
if err != nil {
return err
}
if _, err := tx.Exec(ctx, statement, encoded); err != nil {
args := []any{encoded}
if table == "resources" {
args = append(args, t.scope.TenantID)
}
tag, err := tx.Exec(ctx, statement, args...)
if err != nil {
return fmt.Errorf("写入生产 %s 失败: %w", table, err)
}
if table == "resources" && tag.RowsAffected() != 1 {
var identity string
_ = json.Unmarshal(row["id"], &identity)
return fmt.Errorf("写入生产 resources 冲突:%s 属于其它租户,拒绝覆盖", identity)
}
}
return nil
}
@@ -663,67 +675,6 @@ func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrele
return nil
}
// dropOrphanResources 清掉"本次恢复后已无任何归属"的目标租户资源行。
//
// 判据必须同时满足:属于目标租户、不在预备份快照里、没有任何 resource_binding、
// 也没有任何分块正文以 resource://<handle> 引用它。这样的行对目标应用完全不可见
// (没有知识库能按它取到文件),只可能是失败写入留下的残渣。
func (t *ProductionLibraryTarget) dropOrphanResources(ctx context.Context, backupSnapshotID string) error {
archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
if err != nil {
return errors.New("归档不可用")
}
defer archive.Close()
manifest, err := archive.Load(backupSnapshotID)
if err != nil {
return errors.New("预备份快照不可读")
}
kept := []string{}
for _, part := range manifest.Parts {
if part.Kind != "database" || part.Name != "resources" {
continue
}
ids, err := archivedResourceIDs(archive, part)
if err != nil {
return err
}
kept = append(kept, ids...)
}
rows, err := t.pool.Query(ctx, `
SELECT r.id FROM resources AS r
WHERE r.tenant_id=$1
AND NOT (r.id = ANY($2::text[]))
AND NOT EXISTS (SELECT 1 FROM resource_bindings b WHERE b.resource_id=r.id)
AND NOT EXISTS (SELECT 1 FROM chunks c WHERE c.tenant_id=r.tenant_id AND c.deleted_at IS NULL
AND coalesce(r.handle,'') <> ''
AND position('resource://'||r.handle IN coalesce(c.content,'')) > 0)`,
t.scope.TenantID, kept)
if err != nil {
return fmt.Errorf("查找目标残留资源失败: %w", err)
}
orphans := make([]string, 0, 8)
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
rows.Close()
return err
}
orphans = append(orphans, id)
}
rows.Close()
if err := rows.Err(); err != nil {
return err
}
if len(orphans) == 0 {
return nil
}
if _, err := t.pool.Exec(ctx, `DELETE FROM resources WHERE tenant_id=$1 AND id = ANY($2::text[])`,
t.scope.TenantID, orphans); err != nil {
return fmt.Errorf("清理目标残留资源失败: %w", err)
}
return nil
}
// archivedResourceState 只核对"归档里列出的这些资源"是否还在且未变。
func (t *ProductionLibraryTarget) archivedResourceState(ctx context.Context, ids []string) (int64, string, error) {
rows, err := t.pool.Query(ctx, `SELECT id FROM resources WHERE id = ANY($1::text[]) ORDER BY id`, ids)
@@ -816,30 +767,59 @@ func archiveRowIdentity(table string, row map[string]json.RawMessage) string {
}
// Restore 用预备份把生产整库恢复回去:走同一条 Apply 路径,可重复执行。
//
// 恢复必须带着"本次写集合 + 写前整实例证据"(写第一笔之前落盘、摘要登记在治理库)。
// 缺它就直接失败:此刻目标已经变了,按现状推断"哪些是残渣"会删掉历史孤儿、
// 同租户其它知识库暂未绑定的资源、被共享的资源——而预备份里没有这些行,删了回不来。
func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error {
if backup.SnapshotID == "" {
return errors.New("恢复需要绑定的预备份快照")
}
if backup.WriteSet == nil {
return errors.New("恢复缺少持久化的写集合记录(作业/目标/预备份绑定):拒绝按目标现状推断残渣,维护保持")
}
if err := backup.WriteSet.Verify(); err != nil {
return fmt.Errorf("写集合记录校验失败,拒绝恢复:%w", err)
}
if backup.WriteSet.BackupSnapshotID != backup.SnapshotID {
return fmt.Errorf("写集合记录绑定的是预备份 %s,不是 %s", backup.WriteSet.BackupSnapshotID, backup.SnapshotID)
}
if backup.WriteSet.TenantID != t.scope.TenantID || backup.WriteSet.KnowledgeBaseID != t.scope.KnowledgeBaseID {
return fmt.Errorf("写集合记录绑定的是 %d/%s,与目标 %d/%s 不一致",
backup.WriteSet.TenantID, backup.WriteSet.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID)
}
for _, stage := range libraryrelease.Stages() {
if err := t.Apply(ctx, stage, backup.SnapshotID); err != nil {
return fmt.Errorf("恢复阶段 %s 失败: %w", stage, err)
}
}
// resources 是多知识库共享表,回放阶段不删它:本次写入新增、恢复后已经
// 没有任何归属的资源行会残留。它们对应用不可见,但属于"多写的"。
if err := t.dropOrphanResources(ctx, backup.SnapshotID); err != nil {
// resources 是多知识库共享表,回放阶段不删它。这里只清"写前不存在 ∧ 由本次写入 ∧
// 不在本次恢复内容里"的精确身份,历史孤儿与其它知识库的资源一律不动。
archiveForCleanup, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
if err != nil {
return errors.New("归档不可用")
}
restoredManifest, err := archiveForCleanup.Load(backup.SnapshotID)
if err != nil {
archiveForCleanup.Close()
return errors.New("预备份快照不可读")
}
restored, err := t.snapshotRowIdentities(archiveForCleanup, restoredManifest)
archiveForCleanup.Close()
if err != nil {
return err
}
// 恢复工具把四个阶段都跑完不算恢复完成:整实例必须回到预备份时刻的样子。
// 只比目标范围会把"写到范围外"的残留当成已恢复(真实事故:恢复后仍多 67 行)。
if len(backup.Universe) > 0 {
after, err := universeFingerprint(ctx, t.pool)
if err != nil {
return fmt.Errorf("恢复后读取整库指纹失败: %w", err)
}
if diff := universeDiff(backup.Universe, after); diff != "" {
return fmt.Errorf("恢复后目标与预备份时刻不一致:%s(很可能有写入逃出了预备份范围,不能当作已恢复)", diff)
}
if err := t.clearJobCreatedRows(ctx, *backup.WriteSet, restored); err != nil {
return err
}
// 恢复工具把四个阶段都跑完不算恢复完成:整实例(含其它知识库的内容)必须回到
// 预备份时刻的样子。没有证据就不许降级跳过——那等于把"未经校验"报成"已恢复"。
after, err := universeFingerprint(ctx, t.pool)
if err != nil {
return fmt.Errorf("恢复后读取整库指纹失败: %w", err)
}
if diff := universeDiff(backup.WriteSet.Universe, after); diff != "" {
return fmt.Errorf("恢复后目标与预备份时刻不一致:%s(很可能有写入逃出了预备份范围,不能当作已恢复)", diff)
}
return nil
}
@@ -108,6 +108,30 @@ func TestLibraryProductionTargetIsolated(t *testing.T) {
t.Fatalf("target: %v", err)
}
defer target.Close()
// 写集合在生产路径上是"写第一笔之前"算好的;这个用例直接调 Apply,所以在这里算,
// 整实例证据就是此刻(任何写入之前)的目标状态。
// 夹具里那条软删除文档不在本快照里(快照只含 doc-safe/chunk-safe/wiki-safe),
// 恢复按目标范围回放会正确地把它清掉;为了让"整实例证据"与恢复结果可比,先删掉它。
// 同时把夹具行的内容对齐到本快照(快照是可读的完整版本形态):整实例内容证据
// 才可能与"恢复完成后的目标"逐字节一致。
for _, statement := range []string{
`DELETE FROM knowledges WHERE id='deleted-doc'`,
`UPDATE knowledges SET file_path='' WHERE id='doc-safe'`,
`UPDATE wiki_pages SET content='wiki', status=NULL WHERE id='wiki-safe'`,
// 快照里这些表是空的:目标范围先与它对齐,否则"整实例内容证据"比的是
// 快照里不存在的行,恢复(正确地)删掉它们后反而被判不一致。
`DELETE FROM resource_bindings`,
`DELETE FROM resources`,
`DELETE FROM wiki_page_issues`,
`DELETE FROM wiki_log_entries`,
`DELETE FROM knowledge_tags`,
`DELETE FROM knowledge_tag_relations`,
} {
if _, err := pool.Exec(ctx, statement); err != nil {
t.Fatal(err)
}
}
initialLedger := planLedgerFor(t, target, first, first)
for _, stage := range []libraryrelease.Stage{libraryrelease.StageDatabase, libraryrelease.StageWiki, libraryrelease.StageIndex} {
if err := target.Apply(ctx, stage, first); err != nil {
@@ -149,7 +173,8 @@ func TestLibraryProductionTargetIsolated(t *testing.T) {
t.Fatalf("cross-scope rows must survive, got %d", others)
}
// 恢复:把第一个快照再应用一次,撤销的内容回来。
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup", SnapshotID: first}); err != nil {
// 恢复必须带着写集合记录(生产路径在写第一笔之前就算好并落盘)。
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup", SnapshotID: first, WriteSet: initialLedger}); err != nil {
t.Fatalf("restore: %v", err)
}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: first}); err != nil {
@@ -193,6 +218,9 @@ func TestLibraryProductionTargetIsolated(t *testing.T) {
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: first}); err == nil {
t.Fatal("verify must detect a missing chunk row")
}
// 这次恢复要用"预备份时刻"的整实例证据(Prepare 已经返回它),不是当前状态:
// 目标刚被故意破坏过,按现状算出来的证据会与恢复结果对不上。
backup.WriteSet = planLedgerWithUniverse(t, target, backup.SnapshotID, backup.SnapshotID, backup.Universe)
if err := target.Restore(ctx, backup); err != nil {
t.Fatalf("restore from backup: %v", err)
}
@@ -298,7 +326,7 @@ func TestLibraryReleaseRunIsolated(t *testing.T) {
t.Fatal(err)
}
now := time.Now().UTC()
request := libraryrelease.Request{Target: "internal", SnapshotID: snapshotID,
request := libraryrelease.Request{JobID: "job-e2e-isolated", Target: "internal", SnapshotID: snapshotID,
ParamsDigest: "digest-e2e", Now: now,
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-e2e",
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
@@ -561,7 +589,7 @@ func TestLibraryCrossScopeSyncRollbackLeavesNoResidueIsolated(t *testing.T) {
t.Fatalf("backup store: %v", err)
}
now := time.Now().UTC()
request := libraryrelease.Request{Target: "internal", SnapshotID: snapshotID,
request := libraryrelease.Request{JobID: "job-cross-scope", Target: "internal", SnapshotID: snapshotID,
ParamsDigest: "digest-cross-scope", Now: now,
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-cross-scope",
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
@@ -729,3 +757,29 @@ func sealCrossScopeFixture(t *testing.T, archive *librarysnapshot.Archive, sourc
}
func formatTenant(tenant uint64) string { return strconv.FormatUint(tenant, 10) }
// planLedgerFor 给"直接调 Apply 再 Restore"的用例造一份真实写集合:
// 生产路径由 libraryrelease.Run 在写第一笔之前完成同样的动作。
func planLedgerFor(t *testing.T, target *ProductionLibraryTarget, snapshotID, backupID string) *libraryrelease.WriteSet {
t.Helper()
return planLedgerWithUniverse(t, target, snapshotID, backupID, nil)
}
func planLedgerWithUniverse(t *testing.T, target *ProductionLibraryTarget, snapshotID, backupID string,
universe map[string]libraryrelease.TableFingerprint) *libraryrelease.WriteSet {
t.Helper()
ctx := context.Background()
if universe == nil {
computed, err := universeFingerprint(ctx, target.Pool())
if err != nil {
t.Fatalf("whole-instance fingerprint: %v", err)
}
universe = computed
}
set, err := target.PlanWriteSet(ctx, libraryrelease.WriteSetPlan{JobID: "job-test-" + snapshotID[:8],
Target: "internal", BackupSnapshotID: backupID, TargetSnapshotID: snapshotID, Universe: universe})
if err != nil {
t.Fatalf("plan write set: %v", err)
}
return &set
}
@@ -13,19 +13,23 @@ import (
"knowledge-governance/internal/librarysnapshot"
)
// universeFingerprint 是"整实例"的知识域指纹:不限租户/知识库的行数 + 身份集合摘要。
// universeFingerprint 是"整实例"的知识域指纹:不限租户/知识库的行数与**逐行内容**摘要。
//
// 为什么不能只比目标范围:恢复只按目标范围删除/回放,写到范围外的残留它看不见
// (真实事故:作业报"已恢复到预备份",目标却多出 4 文件 / 38 分块 / 21 资源 /
// 4 绑定)。整实例指纹让"多写了什么"必然暴露,同时也能证明恢复没有误删别的
// 知识库的行。
// 4 绑定)。整实例指纹让"多写了什么"必然暴露,同时也能证明恢复没有误删、误改
// 别的知识库的行。
//
// 为什么必须是内容而不是身份集合:只比 id 集合时,"另一个知识库的同 ID 行被覆盖
// 写坏"(资源表的 ON CONFLICT 就会这样)在指纹上完全看不出来——数量、身份都一样。
// 因此摘要取自每行 to_jsonb 的 md5,再按身份排序聚合。
func universeFingerprint(ctx context.Context, pool *pgxpool.Pool) (map[string]libraryrelease.TableFingerprint, error) {
result := make(map[string]libraryrelease.TableFingerprint, len(productionLibraryTables))
for _, table := range productionLibraryTables {
identity := identityExpr(table)
statement := fmt.Sprintf(
`SELECT count(*), coalesce(md5(string_agg((%s)::text, E'\n' ORDER BY (%s)::text)), '') FROM %s`,
identity, identity, table)
`SELECT count(*), coalesce(md5(string_agg(md5(to_jsonb(x)::text), E'\n' ORDER BY (%s)::text)), '') FROM %s AS x`,
identity, table)
var fingerprint libraryrelease.TableFingerprint
if err := pool.QueryRow(ctx, statement).Scan(&fingerprint.Count, &fingerprint.Digest); err != nil {
return nil, fmt.Errorf("读取整库 %s 指纹失败: %w", table, err)
@@ -0,0 +1,382 @@
package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
)
// 写集合台账:一次发布的"明确写集合 + 写前存在状态 + 预备份时刻整实例证据"。
//
// 为什么必须是持久记录而不是恢复时现算:
// 恢复发生在写失败之后,目标已经变了。此刻按"目标里没人引用的资源就是残渣"去删,
// 会连历史孤儿、同租户其它知识库暂未绑定的资源一起删掉——而预备份里没有这些行,
// 删了就是不可恢复的数据丢失(真实缺陷)。唯一可靠的口径是写之前就记下
// "这次要写哪些精确 ID、哪些写前就已存在",恢复只删"写前不存在 ∧ 由本次写入"的那些。
// writeSetDirName 是台账在归档根下的固定目录,与快照同级:同一块持久卷,
// 重启后按摘要直接取回。
const writeSetDirName = "writesets"
func writeSetPath(archiveRoot, digest string) string {
return filepath.Join(archiveRoot, writeSetDirName, digest+".json")
}
// PersistWriteSet 把台账按内容寻址落盘(文件名即 sha256),随后只读。
// 幂等:同摘要文件已存在且内容一致就直接返回;内容不一致说明摘要被占用,必须失败。
func PersistWriteSet(archiveRoot string, set *libraryrelease.WriteSet) (string, error) {
if set.Digest == "" {
if err := set.Seal(); err != nil {
return "", err
}
}
if len(set.Universe) == 0 {
return "", errors.New("写集合缺少整实例内容证据,拒绝落盘")
}
canonical, err := set.CanonicalBytes()
if err != nil {
return "", err
}
sum := sha256.Sum256(canonical)
digest := hex.EncodeToString(sum[:])
if digest != set.Digest {
return "", errors.New("写集合摘要与内容不符,拒绝落盘")
}
raw, err := json.Marshal(*set)
if err != nil {
return "", fmt.Errorf("编码写集合失败: %w", err)
}
directory := filepath.Join(archiveRoot, writeSetDirName)
if err := os.MkdirAll(directory, 0700); err != nil {
return "", fmt.Errorf("创建写集合目录失败: %w", err)
}
path := writeSetPath(archiveRoot, digest)
if existing, readErr := os.ReadFile(path); readErr == nil {
var recorded libraryrelease.WriteSet
if json.Unmarshal(existing, &recorded) != nil || recorded.Verify() != nil || recorded.Digest != digest {
return "", errors.New("写集合记录同名但内容不符,拒绝覆盖")
}
return path, nil
} else if !errors.Is(readErr, os.ErrNotExist) {
return "", fmt.Errorf("读取已有写集合失败: %w", readErr)
}
temporary := path + ".tmp"
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return "", fmt.Errorf("写入写集合失败: %w", err)
}
if _, err := file.Write(raw); err != nil {
file.Close()
os.Remove(temporary)
return "", fmt.Errorf("写入写集合失败: %w", err)
}
if err := file.Sync(); err != nil {
file.Close()
os.Remove(temporary)
return "", fmt.Errorf("写集合未落盘: %w", err)
}
if err := file.Close(); err != nil {
os.Remove(temporary)
return "", fmt.Errorf("写集合未落盘: %w", err)
}
if err := os.Chmod(temporary, 0444); err != nil {
os.Remove(temporary)
return "", fmt.Errorf("写集合只读化失败: %w", err)
}
if err := os.Rename(temporary, path); err != nil {
os.Remove(temporary)
return "", fmt.Errorf("写集合落盘失败: %w", err)
}
return path, nil
}
// LoadWriteSet 按摘要取回台账:文件名、字节摘要、内容摘要三者必须一致。
// 任何一步不符都返回错误——治理库里的引用指向的就是这个摘要。
func LoadWriteSet(archiveRoot, digest string) (*libraryrelease.WriteSet, error) {
if digest == "" {
return nil, errors.New("没有写集合摘要,无法取回记录")
}
path := writeSetPath(archiveRoot, digest)
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("写集合记录缺失(%s)", digest)
}
var set libraryrelease.WriteSet
if err := json.Unmarshal(raw, &set); err != nil {
return nil, errors.New("写集合记录不可解析")
}
if err := set.Verify(); err != nil {
return nil, err
}
if set.Digest != digest {
return nil, errors.New("写集合记录摘要不符")
}
return &set, nil
}
// PlanWriteSet 在写第一笔之前算出"这次明确要写什么",拒绝跨范围 ID 冲突,
// 并把台账落盘。返回的记录已 Seal 过;调用方负责把摘要登记到治理库。
func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
if plan.JobID == "" || plan.BackupSnapshotID == "" || plan.TargetSnapshotID == "" {
return libraryrelease.WriteSet{}, errors.New("写集合缺少作业/预备份/快照身份")
}
if len(plan.Universe) == 0 {
return libraryrelease.WriteSet{}, errors.New("预备份没有整实例内容证据,拒绝在没有保护证据的情况下写目标")
}
archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
if err != nil {
return libraryrelease.WriteSet{}, errors.New("归档不可用")
}
defer archive.Close()
manifest, err := archive.Load(plan.TargetSnapshotID)
if err != nil {
return libraryrelease.WriteSet{}, errors.New("要同步的快照不可读")
}
if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil {
return libraryrelease.WriteSet{}, err
}
backupResources, err := archivedSnapshotResourceIDs(archive, plan.BackupSnapshotID)
if err != nil {
return libraryrelease.WriteSet{}, err
}
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: plan.JobID, Target: plan.Target,
TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID,
BackupSnapshotID: plan.BackupSnapshotID, TargetSnapshotID: plan.TargetSnapshotID,
CreatedAt: t.now().UTC(), Universe: plan.Universe}
for _, part := range manifest.Parts {
if part.Kind != "database" && part.Kind != "wiki" {
continue
}
ids, err := archivedRowIdentities(archive, part)
if err != nil {
return libraryrelease.WriteSet{}, err
}
if len(ids) == 0 {
continue
}
owners, err := t.existingRowOwners(ctx, part.Name, ids)
if err != nil {
return libraryrelease.WriteSet{}, err
}
for _, id := range ids {
owner, exists := owners[id]
if !exists {
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id})
continue
}
if err := t.assertWriteSetOwnership(part.Name, id, owner, backupResources); err != nil {
return libraryrelease.WriteSet{}, err
}
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id, Existing: true, Owner: owner})
}
}
if err := set.Seal(); err != nil {
return libraryrelease.WriteSet{}, err
}
if _, err := PersistWriteSet(t.config.ArchiveRoot, &set); err != nil {
return libraryrelease.WriteSet{}, err
}
return set, nil
}
// archivedSnapshotResourceIDs 读一份快照里"属于目标自己的资源身份"。
// 资源表被多个知识库共享,只有在这份预备份里的资源行才允许被覆盖写。
func archivedSnapshotResourceIDs(archive *librarysnapshot.Archive, snapshotID string) (map[string]bool, error) {
ids := map[string]bool{}
if snapshotID == "" {
return ids, nil
}
manifest, err := archive.Load(snapshotID)
if err != nil {
return nil, errors.New("预备份快照不可读")
}
for _, part := range manifest.Parts {
if part.Kind != "database" || part.Name != "resources" {
continue
}
values, err := archivedResourceIDs(archive, part)
if err != nil {
return nil, err
}
for _, id := range values {
ids[id] = true
}
}
return ids, nil
}
// existingRowOwners 读出这些身份在目标里的写前归属("<tenant>/<kb>")。
// 没有 id 列的关系表按父表 knowledges 判归属。
func (t *ProductionLibraryTarget) existingRowOwners(ctx context.Context, table string, ids []string) (map[string]string, error) {
owners := map[string]string{}
if table == "knowledge_tag_relations" {
rows, err := t.pool.Query(ctx, `
SELECT r.knowledge_id, coalesce(r.tag_id,''), coalesce(k.tenant_id,0), coalesce(k.knowledge_base_id,'')
FROM knowledge_tag_relations r LEFT JOIN knowledges k ON k.id=r.knowledge_id
WHERE r.knowledge_id = ANY($1::text[])`, ids)
if err != nil {
return nil, fmt.Errorf("读取目标 %s 归属失败", table)
}
defer rows.Close()
for rows.Next() {
var knowledge, tag, kb string
var tenant uint64
if err := rows.Scan(&knowledge, &tag, &tenant, &kb); err != nil {
return nil, err
}
owners[knowledge+":"+tag] = fmt.Sprintf("%d/%s", tenant, kb)
}
return owners, rows.Err()
}
statement := fmt.Sprintf(`SELECT id, tenant_id, coalesce(knowledge_base_id,'') FROM %s WHERE id = ANY($1::text[])`, table)
if table == "resource_bindings" {
// 绑定表没有 knowledge_base_id:归属只能走父表,否则"同租户"就会被当成"自己的"。
statement = `SELECT b.id, b.tenant_id,
coalesce(k.knowledge_base_id, c.knowledge_base_id, '')
FROM resource_bindings b
LEFT JOIN knowledges k ON b.owner_type='knowledge' AND k.id=b.owner_id
LEFT JOIN chunks c ON b.owner_type='chunk' AND c.id=b.owner_id
WHERE b.id = ANY($1::text[])`
} else if !tablesWithKB[table] {
statement = fmt.Sprintf(`SELECT id, tenant_id, '' FROM %s WHERE id = ANY($1::text[])`, table)
}
rows, err := t.pool.Query(ctx, statement, ids)
if err != nil {
return nil, fmt.Errorf("读取目标 %s 归属失败", table)
}
defer rows.Close()
for rows.Next() {
var id, kb string
var tenant uint64
if err := rows.Scan(&id, &tenant, &kb); err != nil {
return nil, err
}
owners[id] = fmt.Sprintf("%d/%s", tenant, kb)
}
return owners, rows.Err()
}
// assertWriteSetOwnership 是跨范围 ID 冲突的预检:同 ID 已经是别人的行时,
// 写入会改掉别人的数据(resources 尤其危险:ON CONFLICT 会整行覆盖)。
// 宁可整次拒绝,也不许"覆盖一下看起来也能跑"。
func (t *ProductionLibraryTarget) assertWriteSetOwnership(table, id, owner string, backupResources map[string]bool) error {
target := fmt.Sprintf("%d/%s", t.scope.TenantID, t.scope.KnowledgeBaseID)
if table == "resources" {
// 资源只有租户:同租户不等于同知识库,必须落在预备份范围内才算自己的。
if owner != fmt.Sprintf("%d/", t.scope.TenantID) {
return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner)
}
if !backupResources[id] {
return fmt.Errorf("资源 ID 冲突:%s 存在于目标租户但不在预备份范围内(属于其它知识库),拒绝覆盖", id)
}
return nil
}
if owner != target && owner != fmt.Sprintf("%d/", t.scope.TenantID) {
return fmt.Errorf("%s ID 冲突:%s 已属于 %s,拒绝写入", table, id, owner)
}
return nil
}
// clearJobCreatedRows 只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复快照里"的精确身份,
// 并且对资源额外要求它仍然没有任何绑定/正文引用(作业失败期间被别的知识库接管
// 使用的资源不能删)。
//
// 为什么要减掉"快照里的身份":这份台账可能是为另一次动作算的(例如手工从某份
// 预备份修复目标)。只要这一行就是恢复要回到的内容,它就不是残渣——按台账删掉
// 等于把刚恢复的行再删一次。
func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set libraryrelease.WriteSet,
restored map[string]map[string]bool) error {
// 子表先删:resource_bindings → chunks/knowledges、wiki 子表 → wiki 页 → 资源。
for _, table := range []string{"resource_bindings", "chunks", "knowledges", "knowledge_tag_relations",
"wiki_page_issues", "wiki_log_entries", "wiki_pages", "wiki_folders", "knowledge_tags", "resources"} {
ids := make([]string, 0, 8)
for _, id := range set.CreatedIDs(table) {
if restored[table][id] {
continue
}
ids = append(ids, id)
}
if len(ids) == 0 {
continue
}
statement := fmt.Sprintf(`DELETE FROM %s WHERE (%s)::text = ANY($1::text[])`, table, identityExpr(table))
if table == "resources" {
statement += `
AND NOT EXISTS (SELECT 1 FROM resource_bindings b WHERE b.resource_id=resources.id)
AND NOT EXISTS (SELECT 1 FROM chunks c WHERE c.deleted_at IS NULL
AND coalesce(resources.handle,'') <> ''
AND position('resource://'||resources.handle IN coalesce(c.content,'')) > 0)`
}
if _, err := t.pool.Exec(ctx, statement, ids); err != nil {
return fmt.Errorf("清理本次新增的 %s 失败: %w", table, err)
}
}
// 复核:本次新建的行一个都不许剩。资源若因"被引用"而保留,那说明恢复没回到
// 预备份状态,必须报出来让维护保持,而不是静默放过。
leftovers := make([]string, 0, 4)
for _, table := range productionLibraryTables {
ids := make([]string, 0, 8)
for _, id := range set.CreatedIDs(table) {
if restored[table][id] {
continue
}
ids = append(ids, id)
}
if len(ids) == 0 {
continue
}
statement := fmt.Sprintf(`SELECT (%s)::text FROM %s WHERE (%s)::text = ANY($1::text[])`,
identityExpr(table), table, identityExpr(table))
rows, err := t.pool.Query(ctx, statement, ids)
if err != nil {
return fmt.Errorf("复核本次新增的 %s 失败: %w", table, err)
}
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
rows.Close()
return err
}
leftovers = append(leftovers, table+"/"+id)
}
rows.Close()
if err := rows.Err(); err != nil {
return err
}
}
if len(leftovers) > 0 {
sort.Strings(leftovers)
return fmt.Errorf("恢复后仍留下本次写入的 %d 行(%v):目标未回到预备份状态,不能当作已恢复",
len(leftovers), leftovers)
}
return nil
}
// snapshotRowIdentities 读出"恢复要用这份快照"的全部行身份(按表)。
func (t *ProductionLibraryTarget) snapshotRowIdentities(archive *librarysnapshot.Archive,
manifest librarysnapshot.Manifest) (map[string]map[string]bool, error) {
identities := map[string]map[string]bool{}
for _, part := range manifest.Parts {
if part.Kind != "database" && part.Kind != "wiki" {
continue
}
ids, err := archivedRowIdentities(archive, part)
if err != nil {
return nil, err
}
identities[part.Name] = map[string]bool{}
for _, id := range ids {
identities[part.Name][id] = true
}
}
return identities, nil
}
@@ -0,0 +1,685 @@
package v071
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"sort"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
)
// 这一组用例验证的是"失败恢复不许误删目标自己的数据"。
//
// 资源用真实对象存储(MinIO/S3),PostgreSQL 与 Elasticsearch 也是真服务:
// 只有"写失败"是按需注入的(flakyStore),业务不变量不靠桩对象成立。
//
// 为什么必须真实对象存储:旧实现按"目标租户里没有绑定、也没被正文引用"清理资源行,
// 它在内存桩上与真实存储上表现一样(都会删),但资源字节/定位符的改写只有在真实
// bucket 上才能证明"恢复后仍是同一份内容"。桩只用于单元级。
type writeSetFixture struct {
ctx context.Context
pool *pgxpool.Pool
client *http.Client
searchBase string
index string
archiveRoot string
objects *flakyStore
bucket string
}
const (
wsTargetTenant = 10001
wsTargetKB = "kb-target"
wsOtherKB = "kb-other"
wsSourceTenant = 10002
wsSourceKB = "kb-source"
wsForeignTnt = 20000
wsBucket = "kg-wire-test-bucket"
wsHandleTarget = "aaaaaaaaaaaaaaaaaaaaaa"
wsHandleShared = "bbbbbbbbbbbbbbbbbbbbbb"
wsHandleOther = "cccccccccccccccccccccc"
wsHandleOrphan = "dddddddddddddddddddddd"
wsHandleSource = "eeeeeeeeeeeeeeeeeeeeee"
wsHandleForeign = "ffffffffffffffffffffff"
)
func wsLocator(backend, provider, bucket string, tenant uint64, id string) string {
return fmt.Sprintf("storage://%s/%s://%s/%d/%s/a.png", backend, provider, bucket, tenant, id)
}
// flakyStore 在真实 S3 之上按需注入写失败:只让"新对象上传"失败,
// 已存在的对象仍可覆盖(恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏)。
// Open/Stat/Copy 全部透传。
type flakyStore struct {
s3 *S3ObjectStore
failPut atomic.Bool
// failPath 是注入失败的定位符片段:只让"这次要新增的那一个资源"上传失败,
// 恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏。
failPath string
puts atomic.Int64
}
func (s *flakyStore) Put(ctx context.Context, provider, path string, reader io.Reader, size int64) error {
if s.failPut.Load() && (s.failPath == "" || strings.Contains(path, s.failPath)) {
return errors.New("注入的对象写入失败")
}
s.puts.Add(1)
return s.s3.Put(ctx, provider, path, reader, size)
}
func (s *flakyStore) Stat(ctx context.Context, provider, path string) (int64, string, error) {
return s.s3.Stat(ctx, provider, path)
}
func (s *flakyStore) Copy(ctx context.Context, provider, source, target string) error {
return s.s3.Copy(ctx, provider, source, target)
}
func (s *flakyStore) Open(ctx context.Context, provider, path string) (io.ReadCloser, error) {
return s.s3.Open(ctx, provider, path)
}
func openWriteSetFixture(t *testing.T) *writeSetFixture {
t.Helper()
return openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true})
}
// writeSetFixtureOptions 让"被杀掉的子进程"复用父进程已经建好的现场:
// 不重建 schema、不新建索引,也不换归档根(写集合与预备份都落在那一个根下)。
type writeSetFixtureOptions struct {
reset bool
archiveRoot string
index string
}
func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writeSetFixture {
t.Helper()
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
base := os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL")
endpoint := os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT")
access, secret := os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"), os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY")
if dsn == "" || base == "" || endpoint == "" || access == "" || secret == "" ||
os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
t.Skip("requires isolated PG, ES and object storage opt-in")
}
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") {
t.Fatal("isolated database required")
}
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
t.Cleanup(cancel)
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatalf("open isolated PG: %v", err)
}
t.Cleanup(pool.Close)
if options.reset {
if _, err := pool.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
t.Fatalf("reset isolated schema: %v", err)
}
}
schemaStatements := []string{
libraryTestSchema,
// 清掉共用夹具里的示例行:本文件只用自己造的目标/其它租户数据,
// 免得"目标非空"被别的用例的行弄成不确定。
`DELETE FROM knowledges; DELETE FROM chunks; DELETE FROM knowledge_tags;
DELETE FROM knowledge_tag_relations; DELETE FROM resources;
DELETE FROM resource_bindings; DELETE FROM wiki_pages; DELETE FROM wiki_folders;
DELETE FROM wiki_page_issues; DELETE FROM wiki_log_entries`,
`ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published'`,
`ALTER TABLE knowledge_tag_relations ADD COLUMN tenant_id bigint`,
`ALTER TABLE resources ADD COLUMN provider text, ADD COLUMN size bigint, ADD COLUMN storage_backend_id text`,
`ALTER TABLE resources ADD PRIMARY KEY (id)`,
`ALTER TABLE resource_bindings ADD COLUMN relation text`,
`CREATE UNIQUE INDEX wiki_pages_scope_slug ON wiki_pages(tenant_id, knowledge_base_id, slug)`,
`CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`,
`INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + wsBucket + `"}',now())`,
`INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`,
}
if options.reset {
for _, statement := range schemaStatements {
if _, err := pool.Exec(ctx, statement); err != nil {
t.Fatalf("fixture schema: %v", err)
}
}
}
token := make([]byte, 8)
if _, err := rand.Read(token); err != nil {
t.Fatal(err)
}
index := "kg_library_test_" + hex.EncodeToString(token)
client := &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
}}
call := func(method, path, body string, want int) {
t.Helper()
request, err := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body))
if err != nil {
t.Fatal(err)
}
request.Header.Set("Content-Type", "application/json")
response, err := client.Do(request)
if err != nil {
t.Fatalf("fixture HTTP: %v", err)
}
defer response.Body.Close()
if response.StatusCode != want {
t.Fatalf("fixture %s %s => %d, want %d", method, path, response.StatusCode, want)
}
}
if options.index != "" {
index = options.index
}
if options.reset {
call("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"},"chunk_id":{"type":"keyword"},"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":2}}}}`, 200)
}
if options.reset {
t.Cleanup(func() {
call("DELETE", "/"+index, "", 200)
call("HEAD", "/"+index, "", 404)
})
}
root := options.archiveRoot
if root == "" {
root = t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
}
store, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: endpoint,
Region: "us-east-1", AccessKeyID: access, SecretAccessKey: secret, Bucket: wsBucket,
ForcePathStyle: true, MaxObjectBytes: 64 * 1024 * 1024})
if err != nil {
t.Fatalf("object store: %v", err)
}
if err := store.EnsureBucket(ctx); err != nil {
t.Fatalf("ensure bucket: %v", err)
}
return &writeSetFixture{ctx: ctx, pool: pool, client: client, searchBase: base, index: index,
archiveRoot: root, objects: &flakyStore{s3: store}, bucket: wsBucket}
}
// seedTargetData 造"目标本来就有数据"的现场:目标自己的行、同租户另一个知识库的行(含资源)、
// 被两个知识库共享的资源、一个历史孤儿资源、另一个租户的行。
func seedTargetData(t *testing.T, fx *writeSetFixture) {
t.Helper()
ctx := fx.ctx
locator := func(backend string, tenant uint64, id string) string {
return wsLocator(backend, "minio", wsBucket, tenant, id)
}
for _, object := range []struct {
tenant uint64
id string
}{
{10001, "res-target"}, {10001, "res-shared"}, {10001, "res-other"},
{10001, "res-orphan"}, {20000, "res-foreign"},
} {
path := locator("backend-target", object.tenant, object.id)
if err := fx.objects.s3.Put(ctx, "minio", path, bytes.NewReader([]byte("12345678")), 8); err != nil {
t.Fatalf("seed object %s: %v", object.id, err)
}
}
statements := []string{
// 目标自己:文档 + 分块(正文引用目标资源)+ 资源 + 绑定 + Wiki。
`INSERT INTO knowledges VALUES('doc-target',10001,'` + wsTargetKB + `','completed',0,'',NULL)`,
`INSERT INTO chunks VALUES('chunk-target',10001,'` + wsTargetKB + `','doc-target','图:resource://` + wsHandleTarget + `',NULL)`,
`INSERT INTO resources VALUES('res-target','` + wsHandleTarget + `',10001,'` + locator("backend-target", 10001, "res-target") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-target',10001,'res-target','knowledge','doc-target',NULL)`,
`INSERT INTO wiki_pages VALUES('wiki-target',10001,'` + wsTargetKB + `','index','目标 wiki',NULL,'published')`,
// 同租户另一个知识库:数据与资源都必须原样保留。
`INSERT INTO knowledges VALUES('doc-other',10001,'` + wsOtherKB + `','completed',0,'',NULL)`,
`INSERT INTO chunks VALUES('chunk-other',10001,'` + wsOtherKB + `','doc-other','别的知识库正文',NULL)`,
`INSERT INTO resources VALUES('res-other','` + wsHandleOther + `',10001,'` + locator("backend-target", 10001, "res-other") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-other',10001,'res-other','knowledge','doc-other',NULL)`,
// 共享资源:同时被目标知识库与另一个知识库绑定。
`INSERT INTO resources VALUES('res-shared','` + wsHandleShared + `',10001,'` + locator("backend-target", 10001, "res-shared") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-shared-target',10001,'res-shared','knowledge','doc-target',NULL)`,
`INSERT INTO resource_bindings VALUES('bind-shared-other',10001,'res-shared','knowledge','doc-other',NULL)`,
// 历史孤儿:没有任何绑定、也没有正文引用。它不是这次作业写的,绝不能被删。
`INSERT INTO resources VALUES('res-orphan','` + wsHandleOrphan + `',10001,'` + locator("backend-target", 10001, "res-orphan") + `',NULL,'minio',8,'backend-target')`,
// 另一个租户的行。
`INSERT INTO knowledges VALUES('doc-foreign',20000,'kb-foreign','completed',0,'',NULL)`,
`INSERT INTO chunks VALUES('chunk-foreign',20000,'kb-foreign','doc-foreign','外部正文',NULL)`,
`INSERT INTO resources VALUES('res-foreign','` + wsHandleForeign + `',20000,'` + locator("backend-target", 20000, "res-foreign") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-foreign',20000,'res-foreign','knowledge','doc-foreign',NULL)`,
}
for _, statement := range statements {
if _, err := fx.pool.Exec(ctx, statement); err != nil {
t.Fatalf("seed target: %v\n%s", err, statement)
}
}
// 目标索引自洽:目标分块必须有对应文档,否则预备份会(正确地)拒绝。
fx.indexDoc(t, "chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "图:"+wsHandleTarget)
}
func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb, knowledge, content string) {
t.Helper()
body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":[0.1,0.2]}`,
tenant, kb, knowledge, id, id, content)
request, err := http.NewRequestWithContext(fx.ctx, "PUT", fmt.Sprintf("%s/%s/_doc/%s?refresh=true", fx.searchBase, fx.index, id),
strings.NewReader(body))
if err != nil {
t.Fatal(err)
}
request.Header.Set("Content-Type", "application/json")
response, err := fx.client.Do(request)
if err != nil {
t.Fatalf("index doc: %v", err)
}
defer response.Body.Close()
if response.StatusCode != 201 && response.StatusCode != 200 {
t.Fatalf("index doc %s => %d", id, response.StatusCode)
}
}
// sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。
func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string {
t.Helper()
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
if err != nil {
t.Fatal(err)
}
defer archive.Close()
encode := func(value any) []byte {
encoded, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
return encoded
}
parts := map[string][]map[string]any{
"knowledges": {{"id": "doc-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
"parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}},
"chunks": {{"id": "chunk-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
"knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}},
"knowledge_tags": {},
"knowledge_tag_relations": {},
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": wsSourceTenant,
"storage_backend_id": "backend-source", "provider": "oss",
"physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-src"),
"size": int64(8)}},
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": wsSourceTenant,
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}},
"wiki_folders": {},
"wiki_pages": {{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"}},
"wiki_page_issues": {},
"wiki_log_entries": {},
}
if tweak != nil {
tweak(parts)
}
kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database",
"knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database",
"wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"}
manifestParts := make([]librarysnapshot.Part, 0, 14)
names := make([]string, 0, len(parts))
for name := range parts {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
part, err := archive.Put(bytes.NewReader(encode(parts[name])))
if err != nil {
t.Fatal(err)
}
part.Kind, part.Name, part.Count = kinds[name], name, int64(len(parts[name]))
manifestParts = append(manifestParts, part)
}
resourcePart, err := archive.Put(bytes.NewReader([]byte("12345678")))
if err != nil {
t.Fatal(err)
}
resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1
manifestParts = append(manifestParts, resourcePart)
indexPart, err := archive.Put(bytes.NewReader(encode([]map[string]any{{"id": "chunk-src", "source": map[string]any{
"tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src",
"chunk_id": "chunk-src", "source_id": "chunk-src", "content": "来源正文 resource://" + wsHandleSource,
"embedding": []float64{0.1, 0.2}}}})))
if err != nil {
t.Fatal(err)
}
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", 1
manifestParts = append(manifestParts, indexPart)
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
Format: librarysnapshot.Format, Scope: "internal", TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB,
CreatedAt: time.Now().UTC(), Parts: manifestParts,
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}}
id, err := archive.Seal(manifest)
if err != nil {
t.Fatalf("seal source snapshot: %v", err)
}
return id
}
func (fx *writeSetFixture) target(t *testing.T) *ProductionLibraryTarget {
t.Helper()
target, err := NewProductionLibraryTarget(fx.ctx, ProductionLibraryConfig{
Postgres: os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), SearchURL: fx.searchBase, SearchIndex: fx.index,
Objects: fx.objects, ObjectsReader: fx.objects, ArchiveRoot: fx.archiveRoot,
ObjectsBucket: fx.bucket, MaxDocuments: 100, HTTPClient: fx.client,
}, ProductionScope{TenantID: wsTargetTenant, KnowledgeBaseID: wsTargetKB})
if err != nil {
t.Fatalf("target: %v", err)
}
t.Cleanup(target.Close)
return target
}
func (fx *writeSetFixture) backups(t *testing.T, target *ProductionLibraryTarget) *ProductionBackupStore {
t.Helper()
mapping, err := target.Search().MappingSHA256(fx.ctx)
if err != nil {
t.Fatal(err)
}
backups, err := NewProductionBackupStore(ProductionBackupConfig{
Pool: target.Pool(), Search: target.Search(), Objects: fx.objects, ArchiveRoot: fx.archiveRoot,
Scope: "internal", Identity: target.Identity(), MaxDocuments: 100,
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
EmbeddingDimension: 1024, IndexMappingSHA256: mapping},
})
if err != nil {
t.Fatalf("backup store: %v", err)
}
return backups
}
// universeDump 是"整实例逐行字节"快照:任何一行被删/被改写都会体现在这里。
func universeDump(t *testing.T, pool *pgxpool.Pool) map[string][]string {
t.Helper()
dump := map[string][]string{}
for _, table := range productionLibraryTables {
rows, err := pool.Query(context.Background(), fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table))
if err != nil {
t.Fatalf("dump %s: %v", table, err)
}
values := make([]string, 0, 32)
for rows.Next() {
var raw string
if err := rows.Scan(&raw); err != nil {
rows.Close()
t.Fatalf("dump scan %s: %v", table, err)
}
values = append(values, raw)
}
rows.Close()
if err := rows.Err(); err != nil {
t.Fatalf("dump %s: %v", table, err)
}
dump[table] = values
}
return dump
}
func diffUniverse(before, after map[string][]string) string {
tables := make([]string, 0, len(before))
for table := range before {
tables = append(tables, table)
}
sort.Strings(tables)
for _, table := range tables {
if len(before[table]) != len(after[table]) {
return fmt.Sprintf("%s 行数 %d -> %d", table, len(before[table]), len(after[table]))
}
for index := range before[table] {
if before[table][index] != after[table][index] {
return fmt.Sprintf("%s 第 %d 行内容变了:%s", table, index, after[table][index])
}
}
}
return ""
}
func requestFor(snapshotID string) libraryrelease.Request {
now := time.Now().UTC()
return libraryrelease.Request{JobID: "job-writeset-acceptance", Target: "internal", SnapshotID: snapshotID,
ParamsDigest: "digest-writeset", Now: now,
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-writeset",
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
}
// 场景:目标非空(目标自己的文档/分块/资源/Wiki)、同租户里还有另一个知识库的数据与资源、
// 一份被两个知识库共享的资源、一个历史孤儿资源、另一个租户的数据;来源快照属于另一个
// 租户/知识库;资源阶段写失败 → 必须回滚,且"目标原有数据逐行字节不变、本次残留为 0"。
func TestLibraryRollbackKeepsPreExistingTargetDataIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
before := universeDump(t, fx.pool)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
fx.objects.failPath = "/res-src/"
fx.objects.failPut.Store(true)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
fx.objects.failPut.Store(false)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
if outcome.FailedAt != libraryrelease.StageResources {
t.Fatalf("failed_at = %s, want resources", outcome.FailedAt)
}
t.Logf("rollback=%s backup=%s ledger=%s", outcome.Rollback, outcome.Backup.SnapshotID,
outcome.Backup.WriteSet.Digest)
after := universeDump(t, fx.pool)
if diff := diffUniverse(before, after); diff != "" {
t.Fatalf("回滚后目标数据被改动:%s", diff)
}
// 历史孤儿不是本次写的:它必须还在(旧实现按"目标租户里没人引用"清理,会删掉它)。
var orphan string
if err := fx.pool.QueryRow(fx.ctx, `SELECT physical_path FROM resources WHERE id='res-orphan'`).Scan(&orphan); err != nil {
t.Fatalf("历史孤儿资源被删了(恢复把'没人引用的目标租户资源'当成本次残渣):%v", err)
}
var sourceScope int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2`,
wsSourceTenant, wsSourceKB).Scan(&sourceScope); err != nil {
t.Fatal(err)
}
if sourceScope != 0 {
t.Fatalf("回滚后仍有 %d 行来源范围文档", sourceScope)
}
var residue int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-src'`).Scan(&residue); err != nil {
t.Fatal(err)
}
if residue != 0 {
t.Fatalf("本次同步新增的资源行未被清理")
}
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
t.Fatalf("回滚必须与预备份一致:%v", err)
}
// 写集合必须已落盘、可校验,并且明确区分"本次新建"与"写前已存在"。
if outcome.Backup.WriteSet == nil || outcome.Backup.WriteSet.Digest == "" {
t.Fatalf("预备份必须带着已落盘的写集合:%+v", outcome.Backup)
}
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
if err != nil {
t.Fatalf("写集合记录取不回来:%v", err)
}
createdResources := ledger.CreatedIDs("resources")
if len(createdResources) != 1 || createdResources[0] != "res-src" {
t.Fatalf("写集合里本次新建的资源 = %v,want [res-src]", createdResources)
}
// 写集合只允许包含"这次要写"的来源身份;目标自己的资源(含共享/历史孤儿/
// 其它知识库的资源)一个都不许出现,否则恢复就会按它去删。
created := map[string]bool{}
for _, id := range ledger.CreatedIDs("resources") {
created[id] = true
}
for _, id := range []string{"res-target", "res-shared", "res-orphan", "res-other", "res-foreign"} {
if created[id] {
t.Fatalf("写集合把目标原有的资源 %s 当成本次新建(恢复会删掉它):%v", id, created)
}
}
if ledger.JobID != "job-writeset-acceptance" || ledger.BackupSnapshotID != outcome.Backup.SnapshotID {
t.Fatalf("写集合没有绑定作业/预备份:%+v", ledger)
}
if ledger.UniverseDigest() == "" {
t.Fatal("写集合必须带整实例内容证据")
}
}
// 场景:恢复只拿到"快照 id"(worker 重启后的真实形状)→ 没有持久化写集合就不许恢复,
// 更不能把校验降级成"跳过整实例比对"。
func TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
fx.objects.failPath = "/res-src/"
fx.objects.failPut.Store(true)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
fx.objects.failPut.Store(false)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
before := universeDump(t, fx.pool)
// 丢掉内存里的写集合:只按快照 id 恢复(这就是 fixedBackupStore 的形状)。
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + outcome.Backup.SnapshotID,
SnapshotID: outcome.Backup.SnapshotID}); err == nil {
t.Fatal("缺少持久化写集合记录时恢复必须失败(不许把校验降级成静默跳过)")
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的恢复不许动目标:%s", diff)
}
}
// 场景:来源快照里的资源 ID 已被另一个租户占用 → 必须在写第一笔之前拒绝,
// 不能 ON CONFLICT 改写别人的行。
func TestLibraryPreflightRejectsCrossScopeIDConflictIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
// 另一个租户里已经有一条同 ID 的资源行。
if _, err := fx.pool.Exec(fx.ctx, `INSERT INTO resources VALUES('res-src','`+wsHandleForeign+`',20000,'`+
wsLocator("backend-foreign", "oss", "foreign-bucket", 20000, "res-src")+`',NULL,'oss',4,'backend-foreign')`); err != nil {
t.Fatal(err)
}
before := universeDump(t, fx.pool)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("outcome = %+v", outcome)
}
if outcome.FailedAt != "" {
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at = %s", outcome.FailedAt)
}
if !strings.Contains(outcome.Error, "冲突") || !strings.Contains(outcome.Error, "res-src") {
t.Fatalf("拒绝原因必须点名冲突的资源 ID:%s", outcome.Error)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("冲突预检失败后目标被改动:%s", diff)
}
t.Logf("conflict rejected before write: %s", outcome.Error)
}
// 场景:wiki 阶段半写失败(resources 已成功上传、database 已提交)→ 回滚必须把
// 本次新增的资源也清掉,目标原有数据逐行不变。
func TestLibraryWikiStageHalfWriteRollbackIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
before := universeDump(t, fx.pool)
// 同一个作用域里两页同 slug:真实 schema 的唯一索引会让 wiki 阶段写失败,
// 而 database/resources 阶段已经成功提交(半写现场)。
snapshotID := sealSourceSnapshot(t, fx, func(parts map[string][]map[string]any) {
parts["wiki_pages"] = []map[string]any{
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
}
})
target := fx.target(t)
backups := fx.backups(t, target)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
if outcome.State != "rolled_back" {
t.Fatalf("outcome = %+v", outcome)
}
if outcome.FailedAt != libraryrelease.StageWiki {
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("wiki 阶段半写回滚后目标数据被改动:%s", diff)
}
if _, _, err := fx.objects.s3.Stat(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src")); err == nil {
t.Log("note: 对象字节保留在桶里(恢复只回滚行,不删对象;对象清理不属于本次范围)")
}
t.Logf("wiki half-write rolled back: %s", outcome.Rollback)
}
// 场景:写集合记录被删除/篡改 → 恢复必须失败并保持维护,不能"看不见记录就当没写过"。
func TestLibraryWriteSetRecordTamperRefusesRestoreIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
fx.objects.failPath = "/res-src/"
fx.objects.failPut.Store(true)
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
fx.objects.failPut.Store(false)
if outcome.State != "rolled_back" || outcome.Backup.WriteSet == nil {
t.Fatalf("outcome = %+v", outcome)
}
digest := outcome.Backup.WriteSet.Digest
path := writeSetPath(fx.archiveRoot, digest)
// 1) 记录被篡改:同一个文件改一个字段 → 取回必须失败(摘要对不上)。
original, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
tampered := bytes.Replace(original, []byte(`"tenant_id":10001`), []byte(`"tenant_id":10002`), 1)
if bytes.Equal(tampered, original) {
t.Fatalf("篡改夹具没生效:%s", original)
}
if err := os.Chmod(path, 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, tampered, 0600); err != nil {
t.Fatal(err)
}
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
t.Fatal("写集合被篡改后取回必须失败")
}
// 内存里带着被篡改的记录去恢复,同样必须失败。
bad := *outcome.Backup.WriteSet
bad.TenantID = 10002
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: outcome.Backup.ID,
SnapshotID: outcome.Backup.SnapshotID, WriteSet: &bad}); err == nil {
t.Fatal("写集合内容与摘要不符时恢复必须失败")
}
// 2) 记录被删除:恢复必须失败并保持维护(不是静默跳过校验)。
if err := os.WriteFile(path, original, 0600); err != nil {
t.Fatal(err)
}
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
t.Fatal("写集合记录被删除后取回必须失败")
}
t.Logf("tampered/removed ledger refused, digest=%s", digest)
}
@@ -0,0 +1,186 @@
package v071
import (
"context"
"encoding/json"
"io"
"os"
"os/exec"
"strings"
"syscall"
"testing"
"time"
"knowledge-governance/internal/libraryrelease"
)
// 真杀进程的接管验证:子进程在 resources 阶段被 SIGKILL(模拟 worker 被杀),
// 已经提交的 database 阶段与"写集合 + 预备份"记录留在盘上;父进程作为"重启后的
// worker"只按持久记录恢复。
//
// 说明边界:杀的是**发布路径的进程**(本文件的 helper),不是治理栈里的 worker 容器;
// 队列/UI 侧的接管由父负责页面验收。这里证明的是"进程没了之后,恢复只用盘上的
// 写集合与整实例证据,并且删不到目标原有数据"。
const (
killChildEnv = "KG_WRITESET_KILL_CHILD"
killStateEnv = "KG_WRITESET_KILL_STATE"
killJournalEnv = "KG_WRITESET_KILL_JOURNAL"
)
// killHelperState 是子进程写下的"治理库记录"替身:真实系统里这两项分别落在
// library_jobs.snapshot_id 与 library_versions.evidence.writeset_digest。
type killHelperState struct {
JobID string `json:"job_id"`
BackupSnapshotID string `json:"backup_snapshot_id"`
TargetSnapshotID string `json:"target_snapshot_id"`
WriteSetDigest string `json:"writeset_digest"`
}
func TestWriteSetKillHelper(t *testing.T) {
if os.Getenv(killChildEnv) != "1" {
t.Skip("helper process only")
}
// 子进程复用父进程的现场:不重建 schema/索引,写集合与预备份仍落在同一个归档根。
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{archiveRoot: os.Getenv(killJournalEnv + "_ARCHIVE"),
index: os.Getenv(killJournalEnv + "_INDEX")})
state := os.Getenv(killStateEnv)
journal := os.Getenv(killJournalEnv)
snapshotID := os.Getenv("KG_WRITESET_KILL_SNAPSHOT")
target := fx.target(t)
backups := fx.backups(t, target)
current, err := target.Current(fx.ctx)
if err != nil {
t.Fatalf("current: %v", err)
}
backup, err := backups.Prepare(fx.ctx, snapshotID, current)
if err != nil {
t.Fatalf("prepare backup: %v", err)
}
set, err := target.PlanWriteSet(fx.ctx, libraryrelease.WriteSetPlan{JobID: "job-kill", Target: "internal",
BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: snapshotID, Universe: backup.Universe})
if err != nil {
t.Fatalf("plan write set: %v", err)
}
record, err := json.Marshal(killHelperState{JobID: set.JobID, BackupSnapshotID: backup.SnapshotID,
TargetSnapshotID: snapshotID, WriteSetDigest: set.Digest})
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(state, record, 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(journal, []byte(fx.archiveRoot), 0600); err != nil {
t.Fatal(err)
}
if err := target.Apply(fx.ctx, libraryrelease.StageDatabase, snapshotID); err != nil {
t.Fatalf("apply database: %v", err)
}
// resources 阶段:第一笔上传就把自己杀掉(真 SIGKILL,没有 defer、没有收尾)。
dying := &killOnPutStore{inner: fx.objects.s3}
target.config.Objects = dying
if err := target.Apply(fx.ctx, libraryrelease.StageResources, snapshotID); err != nil {
t.Fatalf("apply resources: %v", err)
}
t.Fatal("helper should have been killed during the resources stage")
}
// killOnPutStore 在第一次上传时直接 SIGKILL 自己:模拟进程被外部杀掉。
type killOnPutStore struct {
inner *S3ObjectStore
}
func (s *killOnPutStore) Put(_ context.Context, _, _ string, _ io.Reader, _ int64) error {
if err := syscall.Kill(os.Getpid(), syscall.SIGKILL); err != nil {
return err
}
select {}
}
func (s *killOnPutStore) Stat(ctx context.Context, provider, path string) (int64, string, error) {
return s.inner.Stat(ctx, provider, path)
}
func (s *killOnPutStore) Copy(ctx context.Context, provider, source, target string) error {
return s.inner.Copy(ctx, provider, source, target)
}
func TestLibraryKillWorkerThenResumeRecoversWriteSetIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
before := universeDump(t, fx.pool)
snapshotID := sealSourceSnapshot(t, fx, nil)
statePath := t.TempDir() + "/kill.json"
journalPath := t.TempDir() + "/journal.txt"
command := exec.Command(os.Args[0], "-test.run=TestWriteSetKillHelper", "-test.v")
command.Env = append(os.Environ(), killChildEnv+"=1", killStateEnv+"="+statePath,
killJournalEnv+"="+journalPath, killJournalEnv+"_ARCHIVE="+fx.archiveRoot,
killJournalEnv+"_INDEX="+fx.index, "KG_WRITESET_KILL_SNAPSHOT="+snapshotID)
output, err := command.CombinedOutput()
if err == nil {
t.Fatalf("helper must be killed, it exited cleanly: %s", output)
}
exitErr, ok := err.(*exec.ExitError)
if !ok {
t.Fatalf("helper failed to run: %v\n%s", err, output)
}
status, ok := exitErr.Sys().(syscall.WaitStatus)
if !ok || !status.Signaled() || status.Signal() != syscall.SIGKILL {
t.Fatalf("helper must die from SIGKILL, got %v\n%s", err, output)
}
t.Logf("helper killed by %s (half-written target)", status.Signal())
raw, err := os.ReadFile(statePath)
if err != nil {
t.Fatalf("child did not persist the job record: %v", err)
}
var record killHelperState
if err := json.Unmarshal(raw, &record); err != nil {
t.Fatal(err)
}
if record.WriteSetDigest == "" || record.BackupSnapshotID == "" {
t.Fatalf("child must persist writeset digest + backup snapshot: %+v", record)
}
journalRoot, err := os.ReadFile(journalPath)
if err != nil {
t.Fatal(err)
}
if strings.TrimSpace(string(journalRoot)) != fx.archiveRoot {
t.Fatalf("journal root mismatch: %q", journalRoot)
}
// 重启后的 worker:新连接、新目标对象,只拿盘上的记录。
resumedTarget := fx.target(t)
ledger, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest)
if err != nil {
t.Fatalf("重启后取回写集合失败:%v", err)
}
if ledger.JobID != record.JobID || ledger.BackupSnapshotID != record.BackupSnapshotID ||
ledger.TargetSnapshotID != record.TargetSnapshotID {
t.Fatalf("写集合与作业记录不一致:%+v vs %+v", ledger, record)
}
if err := resumedTarget.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + record.BackupSnapshotID,
SnapshotID: record.BackupSnapshotID, WriteSet: ledger}); err != nil {
t.Fatalf("接管恢复失败:%v", err)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("接管恢复后目标数据被改动:%s", diff)
}
if err := resumedTarget.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: record.BackupSnapshotID}); err != nil {
t.Fatalf("接管恢复后必须与预备份一致:%v", err)
}
t.Logf("resumed after SIGKILL: job=%s backup=%s ledger=%s", record.JobID, record.BackupSnapshotID, record.WriteSetDigest)
// 负例:记录被移除 → 恢复必须失败(维护保持),而不是"看不见就当没写过"。
if err := os.Chmod(writeSetPath(fx.archiveRoot, record.WriteSetDigest), 0600); err != nil {
t.Fatal(err)
}
if err := os.Remove(writeSetPath(fx.archiveRoot, record.WriteSetDigest)); err != nil {
t.Fatal(err)
}
if _, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest); err == nil {
t.Fatal("写集合被移除后必须取不回来")
}
_ = time.Now()
}
+35
View File
@@ -187,6 +187,41 @@
"package": "knowledge-governance/internal/qawemail",
"test": "TestScheduledSendRequiresConfirmationAndSendsExactlyOnce",
"reason": "需要一个一次性 PostgreSQL 与测试主密钥(KG_WEEKLY_MAIL_TEST_DSN/KG_WEEKLY_MAIL_TEST_KEY),CI 不提供;本批已在隔离库真实跑通并通过"
},
{
"package": "knowledge-governance/internal/releaseadapter/v071",
"test": "TestLibraryRollbackKeepsPreExistingTargetDataIsolated",
"reason": "需要一次性隔离库 kg_library_test_*、随机 ES 索引与真实对象存储(KG_LIBRARY_TEST_S3_*);验证目标非空/同租户其它知识库/共享资源/历史孤儿/跨租户数据在失败恢复后逐行字节不变,默认 CI 无这些服务,本机已实跑通过"
},
{
"package": "knowledge-governance/internal/releaseadapter/v071",
"test": "TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated",
"reason": "同上环境;验证缺少持久化写集合时恢复必须失败且不动目标,默认 CI 跳过不算验证"
},
{
"package": "knowledge-governance/internal/releaseadapter/v071",
"test": "TestLibraryPreflightRejectsCrossScopeIDConflictIsolated",
"reason": "同上环境;验证跨范围同 ID 资源冲突在任何写入之前被拒绝,默认 CI 跳过不算验证"
},
{
"package": "knowledge-governance/internal/releaseadapter/v071",
"test": "TestLibraryWikiStageHalfWriteRollbackIsolated",
"reason": "同上环境;验证 wiki 阶段半写(database/resources 已提交)后回滚不留残渣,默认 CI 跳过不算验证"
},
{
"package": "knowledge-governance/internal/releaseadapter/v071",
"test": "TestLibraryWriteSetRecordTamperRefusesRestoreIsolated",
"reason": "同上环境;验证写集合记录被篡改/删除后取回与恢复都失败,默认 CI 跳过不算验证"
},
{
"package": "knowledge-governance/internal/releaseadapter/v071",
"test": "TestLibraryKillWorkerThenResumeRecoversWriteSetIsolated",
"reason": "同上环境并用 SIGKILL 杀掉子进程模拟 worker 被杀,验证重启后只按持久写集合恢复且目标原有数据不变;需要本地真实 PG/ES/MinIO,默认 CI 跳过不算验证"
},
{
"package": "knowledge-governance/internal/releaseadapter/v071",
"test": "TestWriteSetKillHelper",
"reason": "上面 SIGKILL 用例的子进程 fixture,单独运行没有意义;默认 CI 跳过"
}
]
}