失败恢复会在写完之后按"目标租户里没人引用"清理资源行,于是历史孤儿、同租户其它 知识库暂未绑定的资源、被共享的资源都可能被删掉;而预备份里没有这些行,删了不可 恢复,随后整实例指纹才发现不一致,只能停在 rollback_failed。恢复校验依赖的整实例 证据又只存在内存里,worker 重启后只按快照 id 恢复,等于静默跳过比对。 改法:写第一笔之前把"本次明确写集合 + 写前存在状态 + 预备份时刻整实例内容证据" 算出来并按内容寻址落盘(文件名与治理库里的引用都是 sha256),摘要登记在预备份 版本证据里;恢复只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复内容里"的精确 ID,并对 资源额外要求仍无绑定/正文引用;缺记录、记录被改、绑定对不上都直接失败、维护保持。 resources 的 upsert 补租户限定,跨范围同 ID 在写入之前就被预检拒绝。 整实例指纹从"身份集合"改成"逐行内容摘要",别的知识库同 ID 行被覆盖写坏也会暴露。 本地验证(已跑): - make verify 15 项全 success(identity 因本地档案 skipped) - go test ./internal/releaseadapter/v071/ -run Isolated(真实 PG + 真实 ES + 真实 MinIO 对象存储,写失败按定位符注入):目标非空/同租户其它知识库/共享资源/ 历史孤儿/跨租户数据在失败恢复后逐行字节不变、本次残留 0;缺写集合拒绝恢复; 跨范围 ID 冲突在写入前拒绝;wiki 阶段半写回滚干净 - SIGKILL 子进程后按盘上写集合接管恢复,目标原有数据逐行不变 - go test ./internal/libraryrelease/:写集合先于第一笔、规划失败/登记失败都不写目标、 接管不重新规划
This commit is contained in:
@@ -28,6 +28,7 @@ import (
|
||||
"knowledge-governance/internal/libraryfreeze"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarystack"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
type drillVerification struct {
|
||||
@@ -220,7 +221,21 @@ WHERE id = (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$
|
||||
log.Printf("drill: drift detected as expected (%s)", drift.Error)
|
||||
|
||||
// 4) 按预备份恢复。
|
||||
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID}); err != nil {
|
||||
// 恢复必须带着写集合记录。演练只改了一行既有内容(canary),没有新建行,
|
||||
// 所以写集合为空;整实例证据用预备份时刻的那一份。
|
||||
identityValue := target.Identity()
|
||||
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
|
||||
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
|
||||
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
|
||||
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
|
||||
if err := ledger.Seal(); err != nil {
|
||||
return "", errors.New("演练写集合生成失败:" + err.Error())
|
||||
}
|
||||
if _, err := v071.PersistWriteSet(config.LibraryArchiveRoot, &ledger); err != nil {
|
||||
return "", errors.New("演练写集合落盘失败:" + err.Error())
|
||||
}
|
||||
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
|
||||
WriteSet: &ledger}); err != nil {
|
||||
return "", errors.New("按预备份恢复失败:" + err.Error())
|
||||
}
|
||||
// 5) 恢复后逐类复核:必须与预备份完全一致。
|
||||
|
||||
@@ -291,12 +291,24 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
|
||||
var backupSource libraryrelease.BackupStore = backups
|
||||
if resumed {
|
||||
backupSource = fixedBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
|
||||
SnapshotID: previous.SnapshotID}}
|
||||
// 接管必须带着原来那份写集合:没有它,恢复只能按目标现状推断残渣,
|
||||
// 而那时目标已经被写过(真实缺陷:历史孤儿资源被当成残渣删掉)。
|
||||
set, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
|
||||
payload.LibraryJobID, previous.SnapshotID)
|
||||
if ledgerErr != nil {
|
||||
// 读不到就一笔都不许写:目标可能半写,恢复口径未知。
|
||||
message := "接管时读不回写集合记录,拒绝继续写目标:" + ledgerErr.Error()
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
|
||||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", message, time.Now().UTC())
|
||||
recorder.Stage(ctx, "prebackup", "failed", message)
|
||||
return errors.New(message)
|
||||
}
|
||||
backupSource = ledgerBackupStore{ref: libraryrelease.BackupRef{ID: "backup-" + previous.SnapshotID,
|
||||
SnapshotID: previous.SnapshotID}, write: set}
|
||||
}
|
||||
|
||||
request := libraryrelease.Request{Target: payload.Scope, SnapshotID: payload.TargetSnapshotID,
|
||||
ParamsDigest: digest, Now: now,
|
||||
request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope,
|
||||
SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now,
|
||||
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
|
||||
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
|
||||
@@ -506,12 +518,6 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Stor
|
||||
}
|
||||
|
||||
// fixedBackupStore 在崩溃接管时返回已经存在的预备份,不再新建。
|
||||
type fixedBackupStore struct{ ref libraryrelease.BackupRef }
|
||||
|
||||
func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
|
||||
return s.ref, nil
|
||||
}
|
||||
|
||||
// BackupPrepared 把预备份写进作业**并**登记成版本。任何一步失败都返回错误:
|
||||
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
|
||||
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
|
||||
|
||||
@@ -64,6 +64,16 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config
|
||||
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
|
||||
}
|
||||
prebackup := windowJob.SnapshotID
|
||||
// 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣,
|
||||
// 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。
|
||||
ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope,
|
||||
payload.WindowJobID, prebackup)
|
||||
if ledgerErr != nil {
|
||||
note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error()
|
||||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC())
|
||||
return errors.New(note)
|
||||
}
|
||||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||||
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
|
||||
if wireErr != nil {
|
||||
@@ -78,7 +88,7 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
|
||||
restorer := recoveryRestorer{target: target, expected: prebackup}
|
||||
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger}
|
||||
verifier := recoveryVerifier{target: target, expected: prebackup}
|
||||
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
|
||||
now := time.Now().UTC()
|
||||
@@ -106,13 +116,21 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config
|
||||
type recoveryRestorer struct {
|
||||
target *v071.ProductionLibraryTarget
|
||||
expected string
|
||||
writeSet *libraryrelease.WriteSet
|
||||
}
|
||||
|
||||
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
|
||||
if backupID != r.expected {
|
||||
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
|
||||
}
|
||||
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID})
|
||||
if r.writeSet == nil {
|
||||
return errors.New("恢复缺少持久化的写集合记录:拒绝按目标现状推断残渣")
|
||||
}
|
||||
if r.writeSet.BackupSnapshotID != backupID {
|
||||
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
|
||||
}
|
||||
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID,
|
||||
WriteSet: r.writeSet})
|
||||
}
|
||||
|
||||
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
// 写集合台账在治理侧的锚点:摘要登记在"这次作业的预备份版本"证据里。
|
||||
// 为什么不用新表:版本记录本来就绑定 scope + snapshot + 作业,且 id 由这三者
|
||||
// 决定;把摘要挂上去,重启后按同一身份就能取回,不需要改 schema。
|
||||
|
||||
// writeSetDigestOf 读某次作业/某份预备份登记的写集合摘要。
|
||||
// 读不到就返回错误:缺失必须让调用方失败,不能降级成"没有记录就当没写过"。
|
||||
func writeSetDigestOf(ctx context.Context, store *postgresstore.Store, scope, jobID, snapshotID string) (string, error) {
|
||||
versions, err := store.LibraryVersions(ctx, scope, 200)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("读不到版本记录:%w", err)
|
||||
}
|
||||
for _, version := range versions {
|
||||
if version.Kind != governance.LibraryVersionProductionPrebackup || version.SnapshotID != snapshotID {
|
||||
continue
|
||||
}
|
||||
if version.SourceJobID != jobID {
|
||||
continue
|
||||
}
|
||||
raw, _ := version.Evidence["writeset_digest"].(string)
|
||||
if raw == "" {
|
||||
return "", fmt.Errorf("预备份 %s 没有登记写集合摘要(作业 %s)", snapshotID, jobID)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
return "", fmt.Errorf("找不到作业 %s 的预备份版本记录(快照 %s)", jobID, snapshotID)
|
||||
}
|
||||
|
||||
// loadWriteSetForJob 按"作业 + 目标 + 预备份"取回写集合并逐项核对绑定。
|
||||
// 任何一步不符都返回错误:恢复宁可保持维护,也不许用一份对不上的记录去删目标数据。
|
||||
func loadWriteSetForJob(ctx context.Context, store *postgresstore.Store, archiveRoot, scope, jobID, snapshotID string) (*libraryrelease.WriteSet, error) {
|
||||
digest, err := writeSetDigestOf(ctx, store, scope, jobID, snapshotID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
set, err := v071.LoadWriteSet(archiveRoot, digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if set.JobID != jobID {
|
||||
return nil, fmt.Errorf("写集合记录属于作业 %s,不是 %s", set.JobID, jobID)
|
||||
}
|
||||
if set.Target != scope {
|
||||
return nil, fmt.Errorf("写集合记录属于目标 %s,不是 %s", set.Target, scope)
|
||||
}
|
||||
if set.BackupSnapshotID != snapshotID {
|
||||
return nil, fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", set.BackupSnapshotID, snapshotID)
|
||||
}
|
||||
return set, nil
|
||||
}
|
||||
|
||||
// WriteSetPlanned 把写集合摘要登记进"这次作业的预备份版本"证据。
|
||||
// 它在第一笔写入之前被调用;失败即中止(调用方据此不写目标)。
|
||||
func (r libraryStageRecorder) WriteSetPlanned(ctx context.Context, set libraryrelease.WriteSet) error {
|
||||
versions, err := r.store.LibraryVersions(ctx, r.scope, 200)
|
||||
if err != nil {
|
||||
return fmt.Errorf("读不到现有版本证据:%w", err)
|
||||
}
|
||||
var existing map[string]any
|
||||
for _, version := range versions {
|
||||
if version.Kind == governance.LibraryVersionProductionPrebackup && version.SnapshotID == set.BackupSnapshotID {
|
||||
existing = version.Evidence
|
||||
break
|
||||
}
|
||||
}
|
||||
evidence := map[string]any{}
|
||||
for key, value := range existing {
|
||||
evidence[key] = value
|
||||
}
|
||||
// 保留预备份原有证据(备份字节、索引映射等),只追加写集合锚点。
|
||||
evidence["backup_snapshot"] = set.BackupSnapshotID
|
||||
evidence["job"] = set.JobID
|
||||
if r.mappingDigest != "" {
|
||||
evidence["index_mapping_sha256"] = r.mappingDigest
|
||||
}
|
||||
evidence["writeset_digest"] = set.Digest
|
||||
evidence["writeset_universe_digest"] = set.UniverseDigest()
|
||||
evidence["writeset_target_snapshot"] = set.TargetSnapshotID
|
||||
evidence["writeset_rows"] = len(set.Rows)
|
||||
return r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{
|
||||
ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, set.BackupSnapshotID),
|
||||
Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: set.BackupSnapshotID,
|
||||
SourceJobID: r.jobID, CreatedBy: "worker", Evidence: evidence, CreatedAt: time.Now().UTC(),
|
||||
})
|
||||
}
|
||||
|
||||
// ledgerBackupStore 是"接管/恢复"用的预备份来源:只认已经持久化的那一份,
|
||||
// 并把它绑定的写集合一起带出来。
|
||||
//
|
||||
// 为什么不能像以前那样只返回 snapshot id:那一刻目标已经被写过,恢复若拿不到
|
||||
// 写集合就只能按目标现状推断残渣——那正是会误删历史资源的路径。所以这里取不到
|
||||
// 记录就直接失败,而不是返回一个"没有 Universe 的引用"。
|
||||
type ledgerBackupStore struct {
|
||||
ref libraryrelease.BackupRef
|
||||
write *libraryrelease.WriteSet
|
||||
}
|
||||
|
||||
func (s ledgerBackupStore) Prepare(context.Context, string, libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
|
||||
if s.write == nil {
|
||||
return libraryrelease.BackupRef{}, errors.New("恢复缺少持久化的写集合记录:拒绝在没有可追溯写集合的情况下继续")
|
||||
}
|
||||
ref := s.ref
|
||||
ref.WriteSet = s.write
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
// sortedWriteSetSummary 只用于日志/证据:把本次写集合按表汇总成稳定的短摘要。
|
||||
func sortedWriteSetSummary(set libraryrelease.WriteSet) string {
|
||||
counts := map[string]int{}
|
||||
for _, row := range set.Rows {
|
||||
counts[row.Table]++
|
||||
}
|
||||
tables := make([]string, 0, len(counts))
|
||||
for table := range counts {
|
||||
tables = append(tables, table)
|
||||
}
|
||||
sort.Strings(tables)
|
||||
summary := ""
|
||||
for _, table := range tables {
|
||||
if summary != "" {
|
||||
summary += ","
|
||||
}
|
||||
summary += fmt.Sprintf("%s=%d", table, counts[table])
|
||||
}
|
||||
return summary
|
||||
}
|
||||
@@ -36,6 +36,9 @@ type Grant struct {
|
||||
|
||||
// Request 是一次待执行的生产同步。
|
||||
type Request struct {
|
||||
// JobID 是这次发布的作业身份:写集合记录与预备份引用都绑定它,
|
||||
// worker 重启后凭它找回"这次写了什么"。
|
||||
JobID string
|
||||
Target string
|
||||
SnapshotID string
|
||||
// ParamsDigest 覆盖会影响生产结果的全部参数(目标、快照、排除范围等)。
|
||||
@@ -75,9 +78,11 @@ type BackupRef struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Universe 是预备份时刻的整实例指纹,恢复后逐表比对。空表示这次没有采集
|
||||
// (例如 worker 被杀后用已持久化的快照接管),此时恢复只能靠内容校验兜底。
|
||||
// Universe 是预备份时刻的整实例指纹,恢复后逐表比对。
|
||||
Universe map[string]TableFingerprint `json:"universe,omitempty"`
|
||||
// WriteSet 是本次发布的写集合与整实例内容证据。它在写第一笔之前就落盘并
|
||||
// 登记摘要;恢复必须按它清理,缺了它恢复要失败(不许降级成"跳过比对")。
|
||||
WriteSet *WriteSet `json:"write_set,omitempty"`
|
||||
}
|
||||
|
||||
type Target interface {
|
||||
@@ -203,6 +208,32 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor
|
||||
}
|
||||
recorder.Stage(ctx, "prebackup", "succeeded", "预备份 "+backup.ID)
|
||||
|
||||
// 写集合必须在第一笔写入之前落盘并登记:恢复只按"这次明确写了什么"清理,
|
||||
// 不按目标现状推断残渣(那会把历史孤儿/共享资源当成本次残留删掉)。
|
||||
if backup.WriteSet == nil {
|
||||
if planner, ok := target.(WriteSetPlanner); ok {
|
||||
set, planErr := planner.PlanWriteSet(ctx, WriteSetPlan{JobID: request.JobID, Target: request.Target,
|
||||
BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: request.SnapshotID, Universe: backup.Universe})
|
||||
if planErr != nil {
|
||||
outcome.State = "failed"
|
||||
outcome.Error = "写集合/冲突预检失败,未对生产执行任何写入:" + planErr.Error()
|
||||
recorder.Stage(ctx, "prebackup", "failed", outcome.Error)
|
||||
return outcome
|
||||
}
|
||||
backup.WriteSet = &set
|
||||
outcome.Backup = backup
|
||||
if observer, ok := recorder.(WriteSetObserver); ok {
|
||||
if err := observer.WriteSetPlanned(ctx, set); err != nil {
|
||||
// 硬门禁:记录没落库 = 恢复时不知道写集合 = 一笔都不许写。
|
||||
outcome.State = "failed"
|
||||
outcome.Error = "写集合已落盘但引用无法持久化,未对生产执行任何写入:" + err.Error()
|
||||
recorder.Stage(ctx, "prebackup", "failed", outcome.Error)
|
||||
return outcome
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, stage := range Stages() {
|
||||
recorder.Stage(ctx, string(stage), "running", "")
|
||||
if err = target.Apply(ctx, stage, request.SnapshotID); err != nil {
|
||||
|
||||
@@ -250,3 +250,162 @@ func TestRunReportsRollbackFailureDistinctly(t *testing.T) {
|
||||
t.Fatalf("stage log = %v", recorder.entries)
|
||||
}
|
||||
}
|
||||
|
||||
// writeSetPlanTarget 记录"写集合规划"与阶段写入的相对顺序:写集合必须在第一笔之前。
|
||||
type writeSetPlanTarget struct {
|
||||
*fakeTarget
|
||||
planErr error
|
||||
plans []WriteSetPlan
|
||||
events *[]string
|
||||
ledger *WriteSet
|
||||
restores []BackupRef
|
||||
}
|
||||
|
||||
func (p *writeSetPlanTarget) PlanWriteSet(_ context.Context, plan WriteSetPlan) (WriteSet, error) {
|
||||
*p.events = append(*p.events, "plan")
|
||||
p.plans = append(p.plans, plan)
|
||||
if p.planErr != nil {
|
||||
return WriteSet{}, p.planErr
|
||||
}
|
||||
set := WriteSet{Version: WriteSetVersion, JobID: plan.JobID, Target: plan.Target,
|
||||
TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: plan.BackupSnapshotID,
|
||||
TargetSnapshotID: plan.TargetSnapshotID, CreatedAt: time.Now().UTC(), Universe: plan.Universe,
|
||||
Rows: []WriteSetRow{{Table: "resources", ID: "res-new"}}}
|
||||
if err := set.Seal(); err != nil {
|
||||
return WriteSet{}, err
|
||||
}
|
||||
p.ledger = &set
|
||||
return set, nil
|
||||
}
|
||||
|
||||
func (p *writeSetPlanTarget) Apply(ctx context.Context, stage Stage, snapshot string) error {
|
||||
*p.events = append(*p.events, "apply:"+string(stage))
|
||||
return p.fakeTarget.Apply(ctx, stage, snapshot)
|
||||
}
|
||||
|
||||
func (p *writeSetPlanTarget) Restore(ctx context.Context, backup BackupRef) error {
|
||||
p.restores = append(p.restores, backup)
|
||||
return p.fakeTarget.Restore(ctx, backup)
|
||||
}
|
||||
|
||||
type writeSetRecorder struct {
|
||||
*stageLog
|
||||
events *[]string
|
||||
err error
|
||||
sets []WriteSet
|
||||
}
|
||||
|
||||
func (r *writeSetRecorder) WriteSetPlanned(_ context.Context, set WriteSet) error {
|
||||
*r.events = append(*r.events, "observe")
|
||||
r.sets = append(r.sets, set)
|
||||
return r.err
|
||||
}
|
||||
|
||||
func plannedRun(t *testing.T) (*writeSetPlanTarget, *writeSetRecorder, *[]string, Request, *fakeBackups) {
|
||||
t.Helper()
|
||||
now := time.Now()
|
||||
events := []string{}
|
||||
target := &writeSetPlanTarget{fakeTarget: &fakeTarget{}, events: &events}
|
||||
recorder := &writeSetRecorder{stageLog: &stageLog{}, events: &events}
|
||||
backups := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap", Bytes: 512,
|
||||
CreatedAt: now.UTC(), Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}}
|
||||
request := request(now)
|
||||
return target, recorder, &events, request, backups
|
||||
}
|
||||
|
||||
func TestRunPlansWriteSetBeforeFirstWrite(t *testing.T) {
|
||||
target, recorder, events, request, backups := plannedRun(t)
|
||||
outcome := Run(context.Background(), request, target, backups, recorder)
|
||||
if outcome.State != "succeeded" {
|
||||
t.Fatalf("outcome = %+v", outcome)
|
||||
}
|
||||
if len(target.plans) != 1 {
|
||||
t.Fatalf("write set must be planned exactly once, got %d", len(target.plans))
|
||||
}
|
||||
if got := (*events)[0]; got != "plan" {
|
||||
t.Fatalf("write set must be planned before anything touches the target, first event = %s", got)
|
||||
}
|
||||
if len(recorder.sets) != 1 || recorder.sets[0].Digest == "" {
|
||||
t.Fatalf("planner output must be persisted before writing: %+v", recorder.sets)
|
||||
}
|
||||
if target.plans[0].JobID != request.JobID || target.plans[0].TargetSnapshotID != request.SnapshotID ||
|
||||
target.plans[0].Target != request.Target {
|
||||
t.Fatalf("plan must bind job/target/snapshot: %+v", target.plans[0])
|
||||
}
|
||||
if target.plans[0].BackupSnapshotID != "backup-snap" {
|
||||
t.Fatalf("plan must bind the backup snapshot: %+v", target.plans[0])
|
||||
}
|
||||
if len(*events) < 2 || (*events)[1] != "observe" {
|
||||
t.Fatalf("write set must be persisted right after planning, events=%v", *events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsBeforeWritingWhenWriteSetPlanFails(t *testing.T) {
|
||||
target, recorder, events, request, backups := plannedRun(t)
|
||||
target.planErr = errors.New("资源 ID 冲突:res-x 已属于其它知识库")
|
||||
outcome := Run(context.Background(), request, target, backups, recorder)
|
||||
if outcome.State != "failed" {
|
||||
t.Fatalf("outcome = %+v", outcome)
|
||||
}
|
||||
if !strings.Contains(outcome.Error, "冲突") {
|
||||
t.Fatalf("error must carry the conflict reason: %s", outcome.Error)
|
||||
}
|
||||
if len(target.applied) != 0 {
|
||||
t.Fatalf("no stage may run after a failed preflight: %v", target.applied)
|
||||
}
|
||||
if len(recorder.sets) != 0 {
|
||||
t.Fatal("a failed plan must not be persisted as the write set")
|
||||
}
|
||||
for _, event := range *events {
|
||||
if strings.HasPrefix(event, "apply:") {
|
||||
t.Fatalf("target was written despite a failed preflight: %v", *events)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsBeforeWritingWhenWriteSetCannotBePersisted(t *testing.T) {
|
||||
target, recorder, events, request, backups := plannedRun(t)
|
||||
recorder.err = errors.New("治理库写不进去")
|
||||
outcome := Run(context.Background(), request, target, backups, recorder)
|
||||
if outcome.State != "failed" {
|
||||
t.Fatalf("outcome = %+v", outcome)
|
||||
}
|
||||
if !strings.Contains(outcome.Error, "写集合") {
|
||||
t.Fatalf("error must name the write-set record: %s", outcome.Error)
|
||||
}
|
||||
if len(target.applied) != 0 {
|
||||
t.Fatalf("no stage may run when the write set cannot be persisted: %v", target.applied)
|
||||
}
|
||||
for _, event := range *events {
|
||||
if strings.HasPrefix(event, "apply:") {
|
||||
t.Fatalf("target was written without a persisted write set: %v", *events)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 接管时必须复用持久化的写集合,而不是按当前(已半写的)目标重新规划。
|
||||
func TestRunResumeUsesPersistedWriteSet(t *testing.T) {
|
||||
target, recorder, events, request, backups := plannedRun(t)
|
||||
resumed := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap",
|
||||
WriteSet: &WriteSet{Version: WriteSetVersion, JobID: request.JobID, Target: request.Target,
|
||||
TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: "backup-snap",
|
||||
TargetSnapshotID: request.SnapshotID, CreatedAt: time.Now().UTC(),
|
||||
Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "d"}}}}}
|
||||
backups = resumed
|
||||
target.fakeTarget.applyErr = map[Stage]error{StageResources: errors.New("上传失败")}
|
||||
outcome := Run(context.Background(), request, target, backups, recorder)
|
||||
if outcome.State != "rolled_back" && outcome.State != "rollback_failed" {
|
||||
t.Fatalf("outcome = %+v", outcome)
|
||||
}
|
||||
if len(target.plans) != 0 {
|
||||
t.Fatalf("resume must not re-plan the write set: %+v", target.plans)
|
||||
}
|
||||
if len(target.restores) != 1 || target.restores[0].WriteSet == nil {
|
||||
t.Fatalf("restore must carry the persisted write set: %+v", target.restores)
|
||||
}
|
||||
for _, event := range *events {
|
||||
if event == "plan" {
|
||||
t.Fatalf("resume re-planned against a half-written target: %v", *events)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
package libraryrelease
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WriteSetVersion 是写集合台账的格式版本。读到的版本不认识就拒绝恢复:
|
||||
// 宁可保持维护,也不要用一份"格式变了但看起来还能读"的记录去删目标数据。
|
||||
const WriteSetVersion = 1
|
||||
|
||||
// WriteSetRow 是台账里的一行:本次发布明确要写的精确身份 + 写前状态。
|
||||
//
|
||||
// 为什么需要"写前状态":恢复不能在目标里按"没人引用"推断残渣。历史孤儿资源、
|
||||
// 同租户另一个知识库暂未绑定的资源、被多个知识库共享的资源,都满足"没人引用",
|
||||
// 但它们不是这次作业写的,删掉就是数据丢失(快照里没有这些行,恢复不回来)。
|
||||
// 只有写前不存在、且由本次写进去的行才允许清理。
|
||||
type WriteSetRow struct {
|
||||
Table string `json:"table"`
|
||||
ID string `json:"id"`
|
||||
// Existing 表示写前目标里已经有这一行(属于目标自己的数据,恢复一律不许删)。
|
||||
Existing bool `json:"existing"`
|
||||
// Owner 是写前该行的归属("<tenant>/<knowledge_base>"),不存在时为空。
|
||||
Owner string `json:"owner,omitempty"`
|
||||
}
|
||||
|
||||
// WriteSetPlan 是调用方在写第一笔之前交给目标的信息:绑定作业、目标、预备份与快照,
|
||||
// 以及预备份时刻的整实例证据。
|
||||
type WriteSetPlan struct {
|
||||
JobID string
|
||||
Target string
|
||||
BackupSnapshotID string
|
||||
TargetSnapshotID string
|
||||
Universe map[string]TableFingerprint
|
||||
}
|
||||
|
||||
// WriteSet 是"这一次发布的写集合 + 写前整实例内容证据",内容寻址、可校验、
|
||||
// 与 job/target/backup 绑定。恢复只读它,不读目标现状去推断。
|
||||
type WriteSet struct {
|
||||
Version int `json:"version"`
|
||||
JobID string `json:"job_id"`
|
||||
Target string `json:"target"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
BackupSnapshotID string `json:"backup_snapshot_id"`
|
||||
TargetSnapshotID string `json:"target_snapshot_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Rows []WriteSetRow `json:"rows"`
|
||||
Universe map[string]TableFingerprint `json:"universe"`
|
||||
// Digest 是对除本字段外全部内容的 sha256:落盘文件名与治理库里的引用都用它,
|
||||
// 记录被改一个字节就对不上。
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// Seal 把行序规范化为可复现的顺序并写入摘要。调用方在落盘与登记之前必须先 Seal。
|
||||
// 排序在这里落地(而不是只在算摘要时排),否则落盘字节与摘要覆盖的内容不一致。
|
||||
func (w *WriteSet) Seal() error {
|
||||
sort.Slice(w.Rows, func(i, j int) bool {
|
||||
if w.Rows[i].Table != w.Rows[j].Table {
|
||||
return w.Rows[i].Table < w.Rows[j].Table
|
||||
}
|
||||
return w.Rows[i].ID < w.Rows[j].ID
|
||||
})
|
||||
digest, err := w.computeDigest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.Digest = digest
|
||||
return nil
|
||||
}
|
||||
|
||||
// CanonicalBytes 是不含摘要字段的规范字节:摘要覆盖的就是它,落盘文件另外
|
||||
// 带上摘要本身(便于自描述),读回时用同一份规范字节重新推导。
|
||||
func (w WriteSet) CanonicalBytes() ([]byte, error) {
|
||||
copied := w
|
||||
copied.Digest = ""
|
||||
rows := make([]WriteSetRow, len(w.Rows))
|
||||
copy(rows, w.Rows)
|
||||
// 行序不影响语义但影响摘要:两边都按 (table,id) 排序才能算出同一个值。
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
if rows[i].Table != rows[j].Table {
|
||||
return rows[i].Table < rows[j].Table
|
||||
}
|
||||
return rows[i].ID < rows[j].ID
|
||||
})
|
||||
copied.Rows = rows
|
||||
raw, err := json.Marshal(copied)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("编码写集合失败: %w", err)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
func (w *WriteSet) computeDigest() (string, error) {
|
||||
raw, err := w.CanonicalBytes()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// Verify 校验摘要与必备身份。缺任一身份都必须失败:没有绑定就没有"这份记录属于
|
||||
// 这个作业/目标/预备份"的证明,拿它去删目标数据无法追责。
|
||||
func (w *WriteSet) Verify() error {
|
||||
if w.Version != WriteSetVersion {
|
||||
return fmt.Errorf("写集合记录版本 %d 不受支持(当前 %d)", w.Version, WriteSetVersion)
|
||||
}
|
||||
if w.JobID == "" || w.BackupSnapshotID == "" || w.TargetSnapshotID == "" {
|
||||
return errors.New("写集合记录缺少作业/预备份/快照绑定")
|
||||
}
|
||||
if w.TenantID == 0 || w.KnowledgeBaseID == "" {
|
||||
return errors.New("写集合记录缺少目标范围身份")
|
||||
}
|
||||
if len(w.Universe) == 0 {
|
||||
return errors.New("写集合记录缺少整实例内容证据")
|
||||
}
|
||||
if w.Digest == "" {
|
||||
return errors.New("写集合记录没有摘要")
|
||||
}
|
||||
digest, err := w.computeDigest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if digest != w.Digest {
|
||||
return errors.New("写集合记录摘要不符(内容被改动或截断)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreatedIDs 返回"本次新建"的精确身份(写前不存在的行)。恢复只允许删这些。
|
||||
func (w WriteSet) CreatedIDs(table string) []string {
|
||||
ids := make([]string, 0, 8)
|
||||
for _, row := range w.Rows {
|
||||
if row.Table == table && !row.Existing {
|
||||
ids = append(ids, row.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
// ExistingIDs 返回写前已存在的身份(用于恢复后复核"目标自己的行都还在")。
|
||||
func (w WriteSet) ExistingIDs(table string) []string {
|
||||
ids := make([]string, 0, 8)
|
||||
for _, row := range w.Rows {
|
||||
if row.Table == table && row.Existing {
|
||||
ids = append(ids, row.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
// UniverseDigest 是整实例证据的摘要,随台账一起登记,便于独立核对。
|
||||
func (w WriteSet) UniverseDigest() string {
|
||||
if len(w.Universe) == 0 {
|
||||
return ""
|
||||
}
|
||||
tables := make([]string, 0, len(w.Universe))
|
||||
for table := range w.Universe {
|
||||
tables = append(tables, table)
|
||||
}
|
||||
sort.Strings(tables)
|
||||
type pair struct {
|
||||
Count int64 `json:"count"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
ordered := make([]pair, 0, len(tables))
|
||||
for _, table := range tables {
|
||||
ordered = append(ordered, pair{Count: w.Universe[table].Count, Digest: w.Universe[table].Digest})
|
||||
}
|
||||
raw, err := json.Marshal(struct {
|
||||
Tables []string `json:"tables"`
|
||||
Values []pair `json:"values"`
|
||||
}{Tables: tables, Values: ordered})
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// WriteSetPlanner 由能访问目标与归档的实现提供:它在写第一笔之前算出写集合、
|
||||
// 拒绝跨范围 ID 冲突,并把记录落盘。失败即"一笔都不许写"。
|
||||
type WriteSetPlanner interface {
|
||||
PlanWriteSet(ctx context.Context, plan WriteSetPlan) (WriteSet, error)
|
||||
}
|
||||
|
||||
// WriteSetObserver 让调用方在写第一笔之前把写集合摘要持久化到治理库。
|
||||
// 失败同样意味着"一笔都不许写":没有可追溯的写集合,恢复就只能靠推断。
|
||||
type WriteSetObserver interface {
|
||||
WriteSetPlanned(ctx context.Context, set WriteSet) error
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package libraryrelease
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func sampleWriteSet() WriteSet {
|
||||
return WriteSet{Version: WriteSetVersion, JobID: "job-1", Target: "internal", TenantID: 10001,
|
||||
KnowledgeBaseID: "kb", BackupSnapshotID: "backup-1", TargetSnapshotID: "snap-1",
|
||||
CreatedAt: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC),
|
||||
Rows: []WriteSetRow{
|
||||
{Table: "resources", ID: "res-new"},
|
||||
{Table: "knowledges", ID: "doc-existing", Existing: true, Owner: "10001/kb"},
|
||||
{Table: "resources", ID: "res-old", Existing: true, Owner: "10001/"},
|
||||
},
|
||||
Universe: map[string]TableFingerprint{"resources": {Count: 2, Digest: "abc"}}}
|
||||
}
|
||||
|
||||
func TestWriteSetSealVerifyRoundTrip(t *testing.T) {
|
||||
set := sampleWriteSet()
|
||||
if err := set.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if set.Digest == "" {
|
||||
t.Fatal("seal must produce a digest")
|
||||
}
|
||||
if err := set.Verify(); err != nil {
|
||||
t.Fatalf("sealed record must verify: %v", err)
|
||||
}
|
||||
// 行序不影响摘要:换序后重新 Sealed 必须得到同一个值(落盘/读回才可比)。
|
||||
shuffled := sampleWriteSet()
|
||||
shuffled.Rows[0], shuffled.Rows[2] = shuffled.Rows[2], shuffled.Rows[0]
|
||||
if err := shuffled.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if shuffled.Digest != set.Digest {
|
||||
t.Fatalf("digest must not depend on row order: %s vs %s", shuffled.Digest, set.Digest)
|
||||
}
|
||||
// 序列化往返(落盘再读回)必须仍然通过校验。
|
||||
raw, err := json.Marshal(shuffled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reloaded WriteSet
|
||||
if err := json.Unmarshal(raw, &reloaded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := reloaded.Verify(); err != nil {
|
||||
t.Fatalf("record read back from disk must verify: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteSetVerifyRejectsTamperingAndMissingEvidence(t *testing.T) {
|
||||
cases := map[string]func(*WriteSet){
|
||||
"内容被改": func(set *WriteSet) {
|
||||
set.Rows[0].ID = "res-other"
|
||||
},
|
||||
"换租户": func(set *WriteSet) {
|
||||
set.TenantID = 20000
|
||||
},
|
||||
"整实例证据被清空": func(set *WriteSet) {
|
||||
set.Universe = nil
|
||||
},
|
||||
"缺预备份绑定": func(set *WriteSet) {
|
||||
set.BackupSnapshotID = ""
|
||||
},
|
||||
"缺摘要": func(set *WriteSet) {
|
||||
set.Digest = ""
|
||||
},
|
||||
"版本不认识": func(set *WriteSet) {
|
||||
set.Version = 99
|
||||
},
|
||||
}
|
||||
for name, mutate := range cases {
|
||||
set := sampleWriteSet()
|
||||
if err := set.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mutate(&set)
|
||||
if err := set.Verify(); err == nil {
|
||||
t.Fatalf("%s:必须校验失败", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteSetCreatedIDsOnlyCoversRowsCreatedByThisRelease(t *testing.T) {
|
||||
set := sampleWriteSet()
|
||||
if err := set.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
created := set.CreatedIDs("resources")
|
||||
if len(created) != 1 || created[0] != "res-new" {
|
||||
t.Fatalf("created resources = %v, want [res-new](写前已存在的资源绝不能被当成残渣)", created)
|
||||
}
|
||||
if strings.Join(set.ExistingIDs("resources"), ",") != "res-old" {
|
||||
t.Fatalf("existing resources = %v", set.ExistingIDs("resources"))
|
||||
}
|
||||
if set.UniverseDigest() == "" {
|
||||
t.Fatal("universe digest must be derived from the whole-instance evidence")
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,11 @@ func TestLibraryExportIsolatedPostgres(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
// 同一个隔离库会被本包的多个用例复用:不清掉上一次的 schema 就会
|
||||
// "relation already exists",让人误以为是导出本身坏了。
|
||||
if _, err = db.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = db.Exec(ctx, libraryTestSchema); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -395,9 +395,11 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc
|
||||
}
|
||||
sort.Strings(columns)
|
||||
// 本库行先删后插:除 resources(可能被多个知识库共享)外不需要 upsert。
|
||||
// resources 的 upsert 必须限定"同租户":同 ID 属于别的租户时整行覆盖会把
|
||||
// 别人的文件改坏(跨范围 ID 冲突已由写集合预检拒绝,这里是最后一道闸)。
|
||||
if table == "resources" {
|
||||
statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb)
|
||||
ON CONFLICT (id) DO UPDATE SET %s`, table, table, updateAssignments(columns))
|
||||
ON CONFLICT (id) DO UPDATE SET %s WHERE %s.tenant_id = $2`, table, table, updateAssignments(columns), table)
|
||||
} else {
|
||||
statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb)`,
|
||||
table, table)
|
||||
@@ -407,9 +409,19 @@ ON CONFLICT (id) DO UPDATE SET %s`, table, table, updateAssignments(columns))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, statement, encoded); err != nil {
|
||||
args := []any{encoded}
|
||||
if table == "resources" {
|
||||
args = append(args, t.scope.TenantID)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, statement, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("写入生产 %s 失败: %w", table, err)
|
||||
}
|
||||
if table == "resources" && tag.RowsAffected() != 1 {
|
||||
var identity string
|
||||
_ = json.Unmarshal(row["id"], &identity)
|
||||
return fmt.Errorf("写入生产 resources 冲突:%s 属于其它租户,拒绝覆盖", identity)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -663,67 +675,6 @@ func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrele
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropOrphanResources 清掉"本次恢复后已无任何归属"的目标租户资源行。
|
||||
//
|
||||
// 判据必须同时满足:属于目标租户、不在预备份快照里、没有任何 resource_binding、
|
||||
// 也没有任何分块正文以 resource://<handle> 引用它。这样的行对目标应用完全不可见
|
||||
// (没有知识库能按它取到文件),只可能是失败写入留下的残渣。
|
||||
func (t *ProductionLibraryTarget) dropOrphanResources(ctx context.Context, backupSnapshotID string) error {
|
||||
archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
|
||||
if err != nil {
|
||||
return errors.New("归档不可用")
|
||||
}
|
||||
defer archive.Close()
|
||||
manifest, err := archive.Load(backupSnapshotID)
|
||||
if err != nil {
|
||||
return errors.New("预备份快照不可读")
|
||||
}
|
||||
kept := []string{}
|
||||
for _, part := range manifest.Parts {
|
||||
if part.Kind != "database" || part.Name != "resources" {
|
||||
continue
|
||||
}
|
||||
ids, err := archivedResourceIDs(archive, part)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kept = append(kept, ids...)
|
||||
}
|
||||
rows, err := t.pool.Query(ctx, `
|
||||
SELECT r.id FROM resources AS r
|
||||
WHERE r.tenant_id=$1
|
||||
AND NOT (r.id = ANY($2::text[]))
|
||||
AND NOT EXISTS (SELECT 1 FROM resource_bindings b WHERE b.resource_id=r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM chunks c WHERE c.tenant_id=r.tenant_id AND c.deleted_at IS NULL
|
||||
AND coalesce(r.handle,'') <> ''
|
||||
AND position('resource://'||r.handle IN coalesce(c.content,'')) > 0)`,
|
||||
t.scope.TenantID, kept)
|
||||
if err != nil {
|
||||
return fmt.Errorf("查找目标残留资源失败: %w", err)
|
||||
}
|
||||
orphans := make([]string, 0, 8)
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
orphans = append(orphans, id)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(orphans) == 0 {
|
||||
return nil
|
||||
}
|
||||
if _, err := t.pool.Exec(ctx, `DELETE FROM resources WHERE tenant_id=$1 AND id = ANY($2::text[])`,
|
||||
t.scope.TenantID, orphans); err != nil {
|
||||
return fmt.Errorf("清理目标残留资源失败: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// archivedResourceState 只核对"归档里列出的这些资源"是否还在且未变。
|
||||
func (t *ProductionLibraryTarget) archivedResourceState(ctx context.Context, ids []string) (int64, string, error) {
|
||||
rows, err := t.pool.Query(ctx, `SELECT id FROM resources WHERE id = ANY($1::text[]) ORDER BY id`, ids)
|
||||
@@ -816,30 +767,59 @@ func archiveRowIdentity(table string, row map[string]json.RawMessage) string {
|
||||
}
|
||||
|
||||
// Restore 用预备份把生产整库恢复回去:走同一条 Apply 路径,可重复执行。
|
||||
//
|
||||
// 恢复必须带着"本次写集合 + 写前整实例证据"(写第一笔之前落盘、摘要登记在治理库)。
|
||||
// 缺它就直接失败:此刻目标已经变了,按现状推断"哪些是残渣"会删掉历史孤儿、
|
||||
// 同租户其它知识库暂未绑定的资源、被共享的资源——而预备份里没有这些行,删了回不来。
|
||||
func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrelease.BackupRef) error {
|
||||
if backup.SnapshotID == "" {
|
||||
return errors.New("恢复需要绑定的预备份快照")
|
||||
}
|
||||
if backup.WriteSet == nil {
|
||||
return errors.New("恢复缺少持久化的写集合记录(作业/目标/预备份绑定):拒绝按目标现状推断残渣,维护保持")
|
||||
}
|
||||
if err := backup.WriteSet.Verify(); err != nil {
|
||||
return fmt.Errorf("写集合记录校验失败,拒绝恢复:%w", err)
|
||||
}
|
||||
if backup.WriteSet.BackupSnapshotID != backup.SnapshotID {
|
||||
return fmt.Errorf("写集合记录绑定的是预备份 %s,不是 %s", backup.WriteSet.BackupSnapshotID, backup.SnapshotID)
|
||||
}
|
||||
if backup.WriteSet.TenantID != t.scope.TenantID || backup.WriteSet.KnowledgeBaseID != t.scope.KnowledgeBaseID {
|
||||
return fmt.Errorf("写集合记录绑定的是 %d/%s,与目标 %d/%s 不一致",
|
||||
backup.WriteSet.TenantID, backup.WriteSet.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||||
}
|
||||
for _, stage := range libraryrelease.Stages() {
|
||||
if err := t.Apply(ctx, stage, backup.SnapshotID); err != nil {
|
||||
return fmt.Errorf("恢复阶段 %s 失败: %w", stage, err)
|
||||
}
|
||||
}
|
||||
// resources 是多知识库共享表,回放阶段不删它:本次写入新增、恢复后已经
|
||||
// 没有任何归属的资源行会残留。它们对应用不可见,但属于"多写的"。
|
||||
if err := t.dropOrphanResources(ctx, backup.SnapshotID); err != nil {
|
||||
// resources 是多知识库共享表,回放阶段不删它。这里只清"写前不存在 ∧ 由本次写入 ∧
|
||||
// 不在本次恢复内容里"的精确身份,历史孤儿与其它知识库的资源一律不动。
|
||||
archiveForCleanup, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
|
||||
if err != nil {
|
||||
return errors.New("归档不可用")
|
||||
}
|
||||
restoredManifest, err := archiveForCleanup.Load(backup.SnapshotID)
|
||||
if err != nil {
|
||||
archiveForCleanup.Close()
|
||||
return errors.New("预备份快照不可读")
|
||||
}
|
||||
restored, err := t.snapshotRowIdentities(archiveForCleanup, restoredManifest)
|
||||
archiveForCleanup.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 恢复工具把四个阶段都跑完不算恢复完成:整实例必须回到预备份时刻的样子。
|
||||
// 只比目标范围会把"写到范围外"的残留当成已恢复(真实事故:恢复后仍多 67 行)。
|
||||
if len(backup.Universe) > 0 {
|
||||
after, err := universeFingerprint(ctx, t.pool)
|
||||
if err != nil {
|
||||
return fmt.Errorf("恢复后读取整库指纹失败: %w", err)
|
||||
}
|
||||
if diff := universeDiff(backup.Universe, after); diff != "" {
|
||||
return fmt.Errorf("恢复后目标与预备份时刻不一致:%s(很可能有写入逃出了预备份范围,不能当作已恢复)", diff)
|
||||
}
|
||||
if err := t.clearJobCreatedRows(ctx, *backup.WriteSet, restored); err != nil {
|
||||
return err
|
||||
}
|
||||
// 恢复工具把四个阶段都跑完不算恢复完成:整实例(含其它知识库的内容)必须回到
|
||||
// 预备份时刻的样子。没有证据就不许降级跳过——那等于把"未经校验"报成"已恢复"。
|
||||
after, err := universeFingerprint(ctx, t.pool)
|
||||
if err != nil {
|
||||
return fmt.Errorf("恢复后读取整库指纹失败: %w", err)
|
||||
}
|
||||
if diff := universeDiff(backup.WriteSet.Universe, after); diff != "" {
|
||||
return fmt.Errorf("恢复后目标与预备份时刻不一致:%s(很可能有写入逃出了预备份范围,不能当作已恢复)", diff)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -108,6 +108,30 @@ func TestLibraryProductionTargetIsolated(t *testing.T) {
|
||||
t.Fatalf("target: %v", err)
|
||||
}
|
||||
defer target.Close()
|
||||
// 写集合在生产路径上是"写第一笔之前"算好的;这个用例直接调 Apply,所以在这里算,
|
||||
// 整实例证据就是此刻(任何写入之前)的目标状态。
|
||||
// 夹具里那条软删除文档不在本快照里(快照只含 doc-safe/chunk-safe/wiki-safe),
|
||||
// 恢复按目标范围回放会正确地把它清掉;为了让"整实例证据"与恢复结果可比,先删掉它。
|
||||
// 同时把夹具行的内容对齐到本快照(快照是可读的完整版本形态):整实例内容证据
|
||||
// 才可能与"恢复完成后的目标"逐字节一致。
|
||||
for _, statement := range []string{
|
||||
`DELETE FROM knowledges WHERE id='deleted-doc'`,
|
||||
`UPDATE knowledges SET file_path='' WHERE id='doc-safe'`,
|
||||
`UPDATE wiki_pages SET content='wiki', status=NULL WHERE id='wiki-safe'`,
|
||||
// 快照里这些表是空的:目标范围先与它对齐,否则"整实例内容证据"比的是
|
||||
// 快照里不存在的行,恢复(正确地)删掉它们后反而被判不一致。
|
||||
`DELETE FROM resource_bindings`,
|
||||
`DELETE FROM resources`,
|
||||
`DELETE FROM wiki_page_issues`,
|
||||
`DELETE FROM wiki_log_entries`,
|
||||
`DELETE FROM knowledge_tags`,
|
||||
`DELETE FROM knowledge_tag_relations`,
|
||||
} {
|
||||
if _, err := pool.Exec(ctx, statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
initialLedger := planLedgerFor(t, target, first, first)
|
||||
|
||||
for _, stage := range []libraryrelease.Stage{libraryrelease.StageDatabase, libraryrelease.StageWiki, libraryrelease.StageIndex} {
|
||||
if err := target.Apply(ctx, stage, first); err != nil {
|
||||
@@ -149,7 +173,8 @@ func TestLibraryProductionTargetIsolated(t *testing.T) {
|
||||
t.Fatalf("cross-scope rows must survive, got %d", others)
|
||||
}
|
||||
// 恢复:把第一个快照再应用一次,撤销的内容回来。
|
||||
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup", SnapshotID: first}); err != nil {
|
||||
// 恢复必须带着写集合记录(生产路径在写第一笔之前就算好并落盘)。
|
||||
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup", SnapshotID: first, WriteSet: initialLedger}); err != nil {
|
||||
t.Fatalf("restore: %v", err)
|
||||
}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: first}); err != nil {
|
||||
@@ -193,6 +218,9 @@ func TestLibraryProductionTargetIsolated(t *testing.T) {
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: first}); err == nil {
|
||||
t.Fatal("verify must detect a missing chunk row")
|
||||
}
|
||||
// 这次恢复要用"预备份时刻"的整实例证据(Prepare 已经返回它),不是当前状态:
|
||||
// 目标刚被故意破坏过,按现状算出来的证据会与恢复结果对不上。
|
||||
backup.WriteSet = planLedgerWithUniverse(t, target, backup.SnapshotID, backup.SnapshotID, backup.Universe)
|
||||
if err := target.Restore(ctx, backup); err != nil {
|
||||
t.Fatalf("restore from backup: %v", err)
|
||||
}
|
||||
@@ -298,7 +326,7 @@ func TestLibraryReleaseRunIsolated(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
request := libraryrelease.Request{Target: "internal", SnapshotID: snapshotID,
|
||||
request := libraryrelease.Request{JobID: "job-e2e-isolated", Target: "internal", SnapshotID: snapshotID,
|
||||
ParamsDigest: "digest-e2e", Now: now,
|
||||
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-e2e",
|
||||
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
|
||||
@@ -561,7 +589,7 @@ func TestLibraryCrossScopeSyncRollbackLeavesNoResidueIsolated(t *testing.T) {
|
||||
t.Fatalf("backup store: %v", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
request := libraryrelease.Request{Target: "internal", SnapshotID: snapshotID,
|
||||
request := libraryrelease.Request{JobID: "job-cross-scope", Target: "internal", SnapshotID: snapshotID,
|
||||
ParamsDigest: "digest-cross-scope", Now: now,
|
||||
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-cross-scope",
|
||||
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
|
||||
@@ -729,3 +757,29 @@ func sealCrossScopeFixture(t *testing.T, archive *librarysnapshot.Archive, sourc
|
||||
}
|
||||
|
||||
func formatTenant(tenant uint64) string { return strconv.FormatUint(tenant, 10) }
|
||||
|
||||
// planLedgerFor 给"直接调 Apply 再 Restore"的用例造一份真实写集合:
|
||||
// 生产路径由 libraryrelease.Run 在写第一笔之前完成同样的动作。
|
||||
func planLedgerFor(t *testing.T, target *ProductionLibraryTarget, snapshotID, backupID string) *libraryrelease.WriteSet {
|
||||
t.Helper()
|
||||
return planLedgerWithUniverse(t, target, snapshotID, backupID, nil)
|
||||
}
|
||||
|
||||
func planLedgerWithUniverse(t *testing.T, target *ProductionLibraryTarget, snapshotID, backupID string,
|
||||
universe map[string]libraryrelease.TableFingerprint) *libraryrelease.WriteSet {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if universe == nil {
|
||||
computed, err := universeFingerprint(ctx, target.Pool())
|
||||
if err != nil {
|
||||
t.Fatalf("whole-instance fingerprint: %v", err)
|
||||
}
|
||||
universe = computed
|
||||
}
|
||||
set, err := target.PlanWriteSet(ctx, libraryrelease.WriteSetPlan{JobID: "job-test-" + snapshotID[:8],
|
||||
Target: "internal", BackupSnapshotID: backupID, TargetSnapshotID: snapshotID, Universe: universe})
|
||||
if err != nil {
|
||||
t.Fatalf("plan write set: %v", err)
|
||||
}
|
||||
return &set
|
||||
}
|
||||
|
||||
@@ -13,19 +13,23 @@ import (
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// universeFingerprint 是"整实例"的知识域指纹:不限租户/知识库的行数 + 身份集合摘要。
|
||||
// universeFingerprint 是"整实例"的知识域指纹:不限租户/知识库的行数与**逐行内容**摘要。
|
||||
//
|
||||
// 为什么不能只比目标范围:恢复只按目标范围删除/回放,写到范围外的残留它看不见
|
||||
// (真实事故:作业报"已恢复到预备份",目标却多出 4 文件 / 38 分块 / 21 资源 /
|
||||
// 4 绑定)。整实例指纹让"多写了什么"必然暴露,同时也能证明恢复没有误删别的
|
||||
// 知识库的行。
|
||||
// 4 绑定)。整实例指纹让"多写了什么"必然暴露,同时也能证明恢复没有误删、误改
|
||||
// 别的知识库的行。
|
||||
//
|
||||
// 为什么必须是内容而不是身份集合:只比 id 集合时,"另一个知识库的同 ID 行被覆盖
|
||||
// 写坏"(资源表的 ON CONFLICT 就会这样)在指纹上完全看不出来——数量、身份都一样。
|
||||
// 因此摘要取自每行 to_jsonb 的 md5,再按身份排序聚合。
|
||||
func universeFingerprint(ctx context.Context, pool *pgxpool.Pool) (map[string]libraryrelease.TableFingerprint, error) {
|
||||
result := make(map[string]libraryrelease.TableFingerprint, len(productionLibraryTables))
|
||||
for _, table := range productionLibraryTables {
|
||||
identity := identityExpr(table)
|
||||
statement := fmt.Sprintf(
|
||||
`SELECT count(*), coalesce(md5(string_agg((%s)::text, E'\n' ORDER BY (%s)::text)), '') FROM %s`,
|
||||
identity, identity, table)
|
||||
`SELECT count(*), coalesce(md5(string_agg(md5(to_jsonb(x)::text), E'\n' ORDER BY (%s)::text)), '') FROM %s AS x`,
|
||||
identity, table)
|
||||
var fingerprint libraryrelease.TableFingerprint
|
||||
if err := pool.QueryRow(ctx, statement).Scan(&fingerprint.Count, &fingerprint.Digest); err != nil {
|
||||
return nil, fmt.Errorf("读取整库 %s 指纹失败: %w", table, err)
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// 写集合台账:一次发布的"明确写集合 + 写前存在状态 + 预备份时刻整实例证据"。
|
||||
//
|
||||
// 为什么必须是持久记录而不是恢复时现算:
|
||||
// 恢复发生在写失败之后,目标已经变了。此刻按"目标里没人引用的资源就是残渣"去删,
|
||||
// 会连历史孤儿、同租户其它知识库暂未绑定的资源一起删掉——而预备份里没有这些行,
|
||||
// 删了就是不可恢复的数据丢失(真实缺陷)。唯一可靠的口径是写之前就记下
|
||||
// "这次要写哪些精确 ID、哪些写前就已存在",恢复只删"写前不存在 ∧ 由本次写入"的那些。
|
||||
|
||||
// writeSetDirName 是台账在归档根下的固定目录,与快照同级:同一块持久卷,
|
||||
// 重启后按摘要直接取回。
|
||||
const writeSetDirName = "writesets"
|
||||
|
||||
func writeSetPath(archiveRoot, digest string) string {
|
||||
return filepath.Join(archiveRoot, writeSetDirName, digest+".json")
|
||||
}
|
||||
|
||||
// PersistWriteSet 把台账按内容寻址落盘(文件名即 sha256),随后只读。
|
||||
// 幂等:同摘要文件已存在且内容一致就直接返回;内容不一致说明摘要被占用,必须失败。
|
||||
func PersistWriteSet(archiveRoot string, set *libraryrelease.WriteSet) (string, error) {
|
||||
if set.Digest == "" {
|
||||
if err := set.Seal(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if len(set.Universe) == 0 {
|
||||
return "", errors.New("写集合缺少整实例内容证据,拒绝落盘")
|
||||
}
|
||||
canonical, err := set.CanonicalBytes()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(canonical)
|
||||
digest := hex.EncodeToString(sum[:])
|
||||
if digest != set.Digest {
|
||||
return "", errors.New("写集合摘要与内容不符,拒绝落盘")
|
||||
}
|
||||
raw, err := json.Marshal(*set)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("编码写集合失败: %w", err)
|
||||
}
|
||||
directory := filepath.Join(archiveRoot, writeSetDirName)
|
||||
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||
return "", fmt.Errorf("创建写集合目录失败: %w", err)
|
||||
}
|
||||
path := writeSetPath(archiveRoot, digest)
|
||||
if existing, readErr := os.ReadFile(path); readErr == nil {
|
||||
var recorded libraryrelease.WriteSet
|
||||
if json.Unmarshal(existing, &recorded) != nil || recorded.Verify() != nil || recorded.Digest != digest {
|
||||
return "", errors.New("写集合记录同名但内容不符,拒绝覆盖")
|
||||
}
|
||||
return path, nil
|
||||
} else if !errors.Is(readErr, os.ErrNotExist) {
|
||||
return "", fmt.Errorf("读取已有写集合失败: %w", readErr)
|
||||
}
|
||||
temporary := path + ".tmp"
|
||||
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("写入写集合失败: %w", err)
|
||||
}
|
||||
if _, err := file.Write(raw); err != nil {
|
||||
file.Close()
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("写入写集合失败: %w", err)
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
file.Close()
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("写集合未落盘: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("写集合未落盘: %w", err)
|
||||
}
|
||||
if err := os.Chmod(temporary, 0444); err != nil {
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("写集合只读化失败: %w", err)
|
||||
}
|
||||
if err := os.Rename(temporary, path); err != nil {
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("写集合落盘失败: %w", err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// LoadWriteSet 按摘要取回台账:文件名、字节摘要、内容摘要三者必须一致。
|
||||
// 任何一步不符都返回错误——治理库里的引用指向的就是这个摘要。
|
||||
func LoadWriteSet(archiveRoot, digest string) (*libraryrelease.WriteSet, error) {
|
||||
if digest == "" {
|
||||
return nil, errors.New("没有写集合摘要,无法取回记录")
|
||||
}
|
||||
path := writeSetPath(archiveRoot, digest)
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("写集合记录缺失(%s)", digest)
|
||||
}
|
||||
var set libraryrelease.WriteSet
|
||||
if err := json.Unmarshal(raw, &set); err != nil {
|
||||
return nil, errors.New("写集合记录不可解析")
|
||||
}
|
||||
if err := set.Verify(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if set.Digest != digest {
|
||||
return nil, errors.New("写集合记录摘要不符")
|
||||
}
|
||||
return &set, nil
|
||||
}
|
||||
|
||||
// PlanWriteSet 在写第一笔之前算出"这次明确要写什么",拒绝跨范围 ID 冲突,
|
||||
// 并把台账落盘。返回的记录已 Seal 过;调用方负责把摘要登记到治理库。
|
||||
func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
|
||||
if plan.JobID == "" || plan.BackupSnapshotID == "" || plan.TargetSnapshotID == "" {
|
||||
return libraryrelease.WriteSet{}, errors.New("写集合缺少作业/预备份/快照身份")
|
||||
}
|
||||
if len(plan.Universe) == 0 {
|
||||
return libraryrelease.WriteSet{}, errors.New("预备份没有整实例内容证据,拒绝在没有保护证据的情况下写目标")
|
||||
}
|
||||
archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSet{}, errors.New("归档不可用")
|
||||
}
|
||||
defer archive.Close()
|
||||
manifest, err := archive.Load(plan.TargetSnapshotID)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSet{}, errors.New("要同步的快照不可读")
|
||||
}
|
||||
if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
backupResources, err := archivedSnapshotResourceIDs(archive, plan.BackupSnapshotID)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: plan.JobID, Target: plan.Target,
|
||||
TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID,
|
||||
BackupSnapshotID: plan.BackupSnapshotID, TargetSnapshotID: plan.TargetSnapshotID,
|
||||
CreatedAt: t.now().UTC(), Universe: plan.Universe}
|
||||
for _, part := range manifest.Parts {
|
||||
if part.Kind != "database" && part.Kind != "wiki" {
|
||||
continue
|
||||
}
|
||||
ids, err := archivedRowIdentities(archive, part)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
continue
|
||||
}
|
||||
owners, err := t.existingRowOwners(ctx, part.Name, ids)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
for _, id := range ids {
|
||||
owner, exists := owners[id]
|
||||
if !exists {
|
||||
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id})
|
||||
continue
|
||||
}
|
||||
if err := t.assertWriteSetOwnership(part.Name, id, owner, backupResources); err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id, Existing: true, Owner: owner})
|
||||
}
|
||||
}
|
||||
if err := set.Seal(); err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
if _, err := PersistWriteSet(t.config.ArchiveRoot, &set); err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
return set, nil
|
||||
}
|
||||
|
||||
// archivedSnapshotResourceIDs 读一份快照里"属于目标自己的资源身份"。
|
||||
// 资源表被多个知识库共享,只有在这份预备份里的资源行才允许被覆盖写。
|
||||
func archivedSnapshotResourceIDs(archive *librarysnapshot.Archive, snapshotID string) (map[string]bool, error) {
|
||||
ids := map[string]bool{}
|
||||
if snapshotID == "" {
|
||||
return ids, nil
|
||||
}
|
||||
manifest, err := archive.Load(snapshotID)
|
||||
if err != nil {
|
||||
return nil, errors.New("预备份快照不可读")
|
||||
}
|
||||
for _, part := range manifest.Parts {
|
||||
if part.Kind != "database" || part.Name != "resources" {
|
||||
continue
|
||||
}
|
||||
values, err := archivedResourceIDs(archive, part)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, id := range values {
|
||||
ids[id] = true
|
||||
}
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// existingRowOwners 读出这些身份在目标里的写前归属("<tenant>/<kb>")。
|
||||
// 没有 id 列的关系表按父表 knowledges 判归属。
|
||||
func (t *ProductionLibraryTarget) existingRowOwners(ctx context.Context, table string, ids []string) (map[string]string, error) {
|
||||
owners := map[string]string{}
|
||||
if table == "knowledge_tag_relations" {
|
||||
rows, err := t.pool.Query(ctx, `
|
||||
SELECT r.knowledge_id, coalesce(r.tag_id,''), coalesce(k.tenant_id,0), coalesce(k.knowledge_base_id,'')
|
||||
FROM knowledge_tag_relations r LEFT JOIN knowledges k ON k.id=r.knowledge_id
|
||||
WHERE r.knowledge_id = ANY($1::text[])`, ids)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取目标 %s 归属失败", table)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var knowledge, tag, kb string
|
||||
var tenant uint64
|
||||
if err := rows.Scan(&knowledge, &tag, &tenant, &kb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owners[knowledge+":"+tag] = fmt.Sprintf("%d/%s", tenant, kb)
|
||||
}
|
||||
return owners, rows.Err()
|
||||
}
|
||||
statement := fmt.Sprintf(`SELECT id, tenant_id, coalesce(knowledge_base_id,'') FROM %s WHERE id = ANY($1::text[])`, table)
|
||||
if table == "resource_bindings" {
|
||||
// 绑定表没有 knowledge_base_id:归属只能走父表,否则"同租户"就会被当成"自己的"。
|
||||
statement = `SELECT b.id, b.tenant_id,
|
||||
coalesce(k.knowledge_base_id, c.knowledge_base_id, '')
|
||||
FROM resource_bindings b
|
||||
LEFT JOIN knowledges k ON b.owner_type='knowledge' AND k.id=b.owner_id
|
||||
LEFT JOIN chunks c ON b.owner_type='chunk' AND c.id=b.owner_id
|
||||
WHERE b.id = ANY($1::text[])`
|
||||
} else if !tablesWithKB[table] {
|
||||
statement = fmt.Sprintf(`SELECT id, tenant_id, '' FROM %s WHERE id = ANY($1::text[])`, table)
|
||||
}
|
||||
rows, err := t.pool.Query(ctx, statement, ids)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取目标 %s 归属失败", table)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id, kb string
|
||||
var tenant uint64
|
||||
if err := rows.Scan(&id, &tenant, &kb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owners[id] = fmt.Sprintf("%d/%s", tenant, kb)
|
||||
}
|
||||
return owners, rows.Err()
|
||||
}
|
||||
|
||||
// assertWriteSetOwnership 是跨范围 ID 冲突的预检:同 ID 已经是别人的行时,
|
||||
// 写入会改掉别人的数据(resources 尤其危险:ON CONFLICT 会整行覆盖)。
|
||||
// 宁可整次拒绝,也不许"覆盖一下看起来也能跑"。
|
||||
func (t *ProductionLibraryTarget) assertWriteSetOwnership(table, id, owner string, backupResources map[string]bool) error {
|
||||
target := fmt.Sprintf("%d/%s", t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||||
if table == "resources" {
|
||||
// 资源只有租户:同租户不等于同知识库,必须落在预备份范围内才算自己的。
|
||||
if owner != fmt.Sprintf("%d/", t.scope.TenantID) {
|
||||
return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner)
|
||||
}
|
||||
if !backupResources[id] {
|
||||
return fmt.Errorf("资源 ID 冲突:%s 存在于目标租户但不在预备份范围内(属于其它知识库),拒绝覆盖", id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if owner != target && owner != fmt.Sprintf("%d/", t.scope.TenantID) {
|
||||
return fmt.Errorf("%s ID 冲突:%s 已属于 %s,拒绝写入", table, id, owner)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// clearJobCreatedRows 只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复快照里"的精确身份,
|
||||
// 并且对资源额外要求它仍然没有任何绑定/正文引用(作业失败期间被别的知识库接管
|
||||
// 使用的资源不能删)。
|
||||
//
|
||||
// 为什么要减掉"快照里的身份":这份台账可能是为另一次动作算的(例如手工从某份
|
||||
// 预备份修复目标)。只要这一行就是恢复要回到的内容,它就不是残渣——按台账删掉
|
||||
// 等于把刚恢复的行再删一次。
|
||||
func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set libraryrelease.WriteSet,
|
||||
restored map[string]map[string]bool) error {
|
||||
// 子表先删:resource_bindings → chunks/knowledges、wiki 子表 → wiki 页 → 资源。
|
||||
for _, table := range []string{"resource_bindings", "chunks", "knowledges", "knowledge_tag_relations",
|
||||
"wiki_page_issues", "wiki_log_entries", "wiki_pages", "wiki_folders", "knowledge_tags", "resources"} {
|
||||
ids := make([]string, 0, 8)
|
||||
for _, id := range set.CreatedIDs(table) {
|
||||
if restored[table][id] {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
continue
|
||||
}
|
||||
statement := fmt.Sprintf(`DELETE FROM %s WHERE (%s)::text = ANY($1::text[])`, table, identityExpr(table))
|
||||
if table == "resources" {
|
||||
statement += `
|
||||
AND NOT EXISTS (SELECT 1 FROM resource_bindings b WHERE b.resource_id=resources.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM chunks c WHERE c.deleted_at IS NULL
|
||||
AND coalesce(resources.handle,'') <> ''
|
||||
AND position('resource://'||resources.handle IN coalesce(c.content,'')) > 0)`
|
||||
}
|
||||
if _, err := t.pool.Exec(ctx, statement, ids); err != nil {
|
||||
return fmt.Errorf("清理本次新增的 %s 失败: %w", table, err)
|
||||
}
|
||||
}
|
||||
// 复核:本次新建的行一个都不许剩。资源若因"被引用"而保留,那说明恢复没回到
|
||||
// 预备份状态,必须报出来让维护保持,而不是静默放过。
|
||||
leftovers := make([]string, 0, 4)
|
||||
for _, table := range productionLibraryTables {
|
||||
ids := make([]string, 0, 8)
|
||||
for _, id := range set.CreatedIDs(table) {
|
||||
if restored[table][id] {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
continue
|
||||
}
|
||||
statement := fmt.Sprintf(`SELECT (%s)::text FROM %s WHERE (%s)::text = ANY($1::text[])`,
|
||||
identityExpr(table), table, identityExpr(table))
|
||||
rows, err := t.pool.Query(ctx, statement, ids)
|
||||
if err != nil {
|
||||
return fmt.Errorf("复核本次新增的 %s 失败: %w", table, err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
leftovers = append(leftovers, table+"/"+id)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(leftovers) > 0 {
|
||||
sort.Strings(leftovers)
|
||||
return fmt.Errorf("恢复后仍留下本次写入的 %d 行(%v):目标未回到预备份状态,不能当作已恢复",
|
||||
len(leftovers), leftovers)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// snapshotRowIdentities 读出"恢复要用这份快照"的全部行身份(按表)。
|
||||
func (t *ProductionLibraryTarget) snapshotRowIdentities(archive *librarysnapshot.Archive,
|
||||
manifest librarysnapshot.Manifest) (map[string]map[string]bool, error) {
|
||||
identities := map[string]map[string]bool{}
|
||||
for _, part := range manifest.Parts {
|
||||
if part.Kind != "database" && part.Kind != "wiki" {
|
||||
continue
|
||||
}
|
||||
ids, err := archivedRowIdentities(archive, part)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
identities[part.Name] = map[string]bool{}
|
||||
for _, id := range ids {
|
||||
identities[part.Name][id] = true
|
||||
}
|
||||
}
|
||||
return identities, nil
|
||||
}
|
||||
@@ -0,0 +1,685 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// 这一组用例验证的是"失败恢复不许误删目标自己的数据"。
|
||||
//
|
||||
// 资源用真实对象存储(MinIO/S3),PostgreSQL 与 Elasticsearch 也是真服务:
|
||||
// 只有"写失败"是按需注入的(flakyStore),业务不变量不靠桩对象成立。
|
||||
//
|
||||
// 为什么必须真实对象存储:旧实现按"目标租户里没有绑定、也没被正文引用"清理资源行,
|
||||
// 它在内存桩上与真实存储上表现一样(都会删),但资源字节/定位符的改写只有在真实
|
||||
// bucket 上才能证明"恢复后仍是同一份内容"。桩只用于单元级。
|
||||
|
||||
type writeSetFixture struct {
|
||||
ctx context.Context
|
||||
pool *pgxpool.Pool
|
||||
client *http.Client
|
||||
searchBase string
|
||||
index string
|
||||
archiveRoot string
|
||||
objects *flakyStore
|
||||
bucket string
|
||||
}
|
||||
|
||||
const (
|
||||
wsTargetTenant = 10001
|
||||
wsTargetKB = "kb-target"
|
||||
wsOtherKB = "kb-other"
|
||||
wsSourceTenant = 10002
|
||||
wsSourceKB = "kb-source"
|
||||
wsForeignTnt = 20000
|
||||
wsBucket = "kg-wire-test-bucket"
|
||||
|
||||
wsHandleTarget = "aaaaaaaaaaaaaaaaaaaaaa"
|
||||
wsHandleShared = "bbbbbbbbbbbbbbbbbbbbbb"
|
||||
wsHandleOther = "cccccccccccccccccccccc"
|
||||
wsHandleOrphan = "dddddddddddddddddddddd"
|
||||
wsHandleSource = "eeeeeeeeeeeeeeeeeeeeee"
|
||||
wsHandleForeign = "ffffffffffffffffffffff"
|
||||
)
|
||||
|
||||
func wsLocator(backend, provider, bucket string, tenant uint64, id string) string {
|
||||
return fmt.Sprintf("storage://%s/%s://%s/%d/%s/a.png", backend, provider, bucket, tenant, id)
|
||||
}
|
||||
|
||||
// flakyStore 在真实 S3 之上按需注入写失败:只让"新对象上传"失败,
|
||||
// 已存在的对象仍可覆盖(恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏)。
|
||||
// Open/Stat/Copy 全部透传。
|
||||
type flakyStore struct {
|
||||
s3 *S3ObjectStore
|
||||
failPut atomic.Bool
|
||||
// failPath 是注入失败的定位符片段:只让"这次要新增的那一个资源"上传失败,
|
||||
// 恢复要把预备份里的对象重新写回去,那条路径不能被注入打坏。
|
||||
failPath string
|
||||
puts atomic.Int64
|
||||
}
|
||||
|
||||
func (s *flakyStore) Put(ctx context.Context, provider, path string, reader io.Reader, size int64) error {
|
||||
if s.failPut.Load() && (s.failPath == "" || strings.Contains(path, s.failPath)) {
|
||||
return errors.New("注入的对象写入失败")
|
||||
}
|
||||
s.puts.Add(1)
|
||||
return s.s3.Put(ctx, provider, path, reader, size)
|
||||
}
|
||||
|
||||
func (s *flakyStore) Stat(ctx context.Context, provider, path string) (int64, string, error) {
|
||||
return s.s3.Stat(ctx, provider, path)
|
||||
}
|
||||
|
||||
func (s *flakyStore) Copy(ctx context.Context, provider, source, target string) error {
|
||||
return s.s3.Copy(ctx, provider, source, target)
|
||||
}
|
||||
|
||||
func (s *flakyStore) Open(ctx context.Context, provider, path string) (io.ReadCloser, error) {
|
||||
return s.s3.Open(ctx, provider, path)
|
||||
}
|
||||
|
||||
func openWriteSetFixture(t *testing.T) *writeSetFixture {
|
||||
t.Helper()
|
||||
return openWriteSetFixtureWith(t, writeSetFixtureOptions{reset: true})
|
||||
}
|
||||
|
||||
// writeSetFixtureOptions 让"被杀掉的子进程"复用父进程已经建好的现场:
|
||||
// 不重建 schema、不新建索引,也不换归档根(写集合与预备份都落在那一个根下)。
|
||||
type writeSetFixtureOptions struct {
|
||||
reset bool
|
||||
archiveRoot string
|
||||
index string
|
||||
}
|
||||
|
||||
func openWriteSetFixtureWith(t *testing.T, options writeSetFixtureOptions) *writeSetFixture {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
|
||||
base := os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL")
|
||||
endpoint := os.Getenv("KG_LIBRARY_TEST_S3_ENDPOINT")
|
||||
access, secret := os.Getenv("KG_LIBRARY_TEST_S3_ACCESS_KEY"), os.Getenv("KG_LIBRARY_TEST_S3_SECRET_KEY")
|
||||
if dsn == "" || base == "" || endpoint == "" || access == "" || secret == "" ||
|
||||
os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
|
||||
t.Skip("requires isolated PG, ES and object storage opt-in")
|
||||
}
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") {
|
||||
t.Fatal("isolated database required")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
|
||||
t.Cleanup(cancel)
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("open isolated PG: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
if options.reset {
|
||||
if _, err := pool.Exec(ctx, `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
|
||||
t.Fatalf("reset isolated schema: %v", err)
|
||||
}
|
||||
}
|
||||
schemaStatements := []string{
|
||||
libraryTestSchema,
|
||||
// 清掉共用夹具里的示例行:本文件只用自己造的目标/其它租户数据,
|
||||
// 免得"目标非空"被别的用例的行弄成不确定。
|
||||
`DELETE FROM knowledges; DELETE FROM chunks; DELETE FROM knowledge_tags;
|
||||
DELETE FROM knowledge_tag_relations; DELETE FROM resources;
|
||||
DELETE FROM resource_bindings; DELETE FROM wiki_pages; DELETE FROM wiki_folders;
|
||||
DELETE FROM wiki_page_issues; DELETE FROM wiki_log_entries`,
|
||||
`ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published'`,
|
||||
`ALTER TABLE knowledge_tag_relations ADD COLUMN tenant_id bigint`,
|
||||
`ALTER TABLE resources ADD COLUMN provider text, ADD COLUMN size bigint, ADD COLUMN storage_backend_id text`,
|
||||
`ALTER TABLE resources ADD PRIMARY KEY (id)`,
|
||||
`ALTER TABLE resource_bindings ADD COLUMN relation text`,
|
||||
`CREATE UNIQUE INDEX wiki_pages_scope_slug ON wiki_pages(tenant_id, knowledge_base_id, slug)`,
|
||||
`CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`,
|
||||
`INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"` + wsBucket + `"}',now())`,
|
||||
`INSERT INTO knowledge_bases VALUES('` + wsTargetKB + `',10001,'{}',NULL),('` + wsOtherKB + `',10001,'{}',NULL),('` + wsSourceKB + `',10002,'{}',NULL)`,
|
||||
}
|
||||
if options.reset {
|
||||
for _, statement := range schemaStatements {
|
||||
if _, err := pool.Exec(ctx, statement); err != nil {
|
||||
t.Fatalf("fixture schema: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
token := make([]byte, 8)
|
||||
if _, err := rand.Read(token); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
index := "kg_library_test_" + hex.EncodeToString(token)
|
||||
client := &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
}}
|
||||
call := func(method, path, body string, want int) {
|
||||
t.Helper()
|
||||
request, err := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
t.Fatalf("fixture HTTP: %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != want {
|
||||
t.Fatalf("fixture %s %s => %d, want %d", method, path, response.StatusCode, want)
|
||||
}
|
||||
}
|
||||
if options.index != "" {
|
||||
index = options.index
|
||||
}
|
||||
if options.reset {
|
||||
call("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"},"chunk_id":{"type":"keyword"},"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":2}}}}`, 200)
|
||||
}
|
||||
if options.reset {
|
||||
t.Cleanup(func() {
|
||||
call("DELETE", "/"+index, "", 200)
|
||||
call("HEAD", "/"+index, "", 404)
|
||||
})
|
||||
}
|
||||
|
||||
root := options.archiveRoot
|
||||
if root == "" {
|
||||
root = t.TempDir()
|
||||
if err := os.Chmod(root, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store, err := NewS3ObjectStore(S3ObjectStoreConfig{Provider: "minio", Endpoint: endpoint,
|
||||
Region: "us-east-1", AccessKeyID: access, SecretAccessKey: secret, Bucket: wsBucket,
|
||||
ForcePathStyle: true, MaxObjectBytes: 64 * 1024 * 1024})
|
||||
if err != nil {
|
||||
t.Fatalf("object store: %v", err)
|
||||
}
|
||||
if err := store.EnsureBucket(ctx); err != nil {
|
||||
t.Fatalf("ensure bucket: %v", err)
|
||||
}
|
||||
return &writeSetFixture{ctx: ctx, pool: pool, client: client, searchBase: base, index: index,
|
||||
archiveRoot: root, objects: &flakyStore{s3: store}, bucket: wsBucket}
|
||||
}
|
||||
|
||||
// seedTargetData 造"目标本来就有数据"的现场:目标自己的行、同租户另一个知识库的行(含资源)、
|
||||
// 被两个知识库共享的资源、一个历史孤儿资源、另一个租户的行。
|
||||
func seedTargetData(t *testing.T, fx *writeSetFixture) {
|
||||
t.Helper()
|
||||
ctx := fx.ctx
|
||||
locator := func(backend string, tenant uint64, id string) string {
|
||||
return wsLocator(backend, "minio", wsBucket, tenant, id)
|
||||
}
|
||||
for _, object := range []struct {
|
||||
tenant uint64
|
||||
id string
|
||||
}{
|
||||
{10001, "res-target"}, {10001, "res-shared"}, {10001, "res-other"},
|
||||
{10001, "res-orphan"}, {20000, "res-foreign"},
|
||||
} {
|
||||
path := locator("backend-target", object.tenant, object.id)
|
||||
if err := fx.objects.s3.Put(ctx, "minio", path, bytes.NewReader([]byte("12345678")), 8); err != nil {
|
||||
t.Fatalf("seed object %s: %v", object.id, err)
|
||||
}
|
||||
}
|
||||
statements := []string{
|
||||
// 目标自己:文档 + 分块(正文引用目标资源)+ 资源 + 绑定 + Wiki。
|
||||
`INSERT INTO knowledges VALUES('doc-target',10001,'` + wsTargetKB + `','completed',0,'',NULL)`,
|
||||
`INSERT INTO chunks VALUES('chunk-target',10001,'` + wsTargetKB + `','doc-target','图:resource://` + wsHandleTarget + `',NULL)`,
|
||||
`INSERT INTO resources VALUES('res-target','` + wsHandleTarget + `',10001,'` + locator("backend-target", 10001, "res-target") + `',NULL,'minio',8,'backend-target')`,
|
||||
`INSERT INTO resource_bindings VALUES('bind-target',10001,'res-target','knowledge','doc-target',NULL)`,
|
||||
`INSERT INTO wiki_pages VALUES('wiki-target',10001,'` + wsTargetKB + `','index','目标 wiki',NULL,'published')`,
|
||||
// 同租户另一个知识库:数据与资源都必须原样保留。
|
||||
`INSERT INTO knowledges VALUES('doc-other',10001,'` + wsOtherKB + `','completed',0,'',NULL)`,
|
||||
`INSERT INTO chunks VALUES('chunk-other',10001,'` + wsOtherKB + `','doc-other','别的知识库正文',NULL)`,
|
||||
`INSERT INTO resources VALUES('res-other','` + wsHandleOther + `',10001,'` + locator("backend-target", 10001, "res-other") + `',NULL,'minio',8,'backend-target')`,
|
||||
`INSERT INTO resource_bindings VALUES('bind-other',10001,'res-other','knowledge','doc-other',NULL)`,
|
||||
// 共享资源:同时被目标知识库与另一个知识库绑定。
|
||||
`INSERT INTO resources VALUES('res-shared','` + wsHandleShared + `',10001,'` + locator("backend-target", 10001, "res-shared") + `',NULL,'minio',8,'backend-target')`,
|
||||
`INSERT INTO resource_bindings VALUES('bind-shared-target',10001,'res-shared','knowledge','doc-target',NULL)`,
|
||||
`INSERT INTO resource_bindings VALUES('bind-shared-other',10001,'res-shared','knowledge','doc-other',NULL)`,
|
||||
// 历史孤儿:没有任何绑定、也没有正文引用。它不是这次作业写的,绝不能被删。
|
||||
`INSERT INTO resources VALUES('res-orphan','` + wsHandleOrphan + `',10001,'` + locator("backend-target", 10001, "res-orphan") + `',NULL,'minio',8,'backend-target')`,
|
||||
// 另一个租户的行。
|
||||
`INSERT INTO knowledges VALUES('doc-foreign',20000,'kb-foreign','completed',0,'',NULL)`,
|
||||
`INSERT INTO chunks VALUES('chunk-foreign',20000,'kb-foreign','doc-foreign','外部正文',NULL)`,
|
||||
`INSERT INTO resources VALUES('res-foreign','` + wsHandleForeign + `',20000,'` + locator("backend-target", 20000, "res-foreign") + `',NULL,'minio',8,'backend-target')`,
|
||||
`INSERT INTO resource_bindings VALUES('bind-foreign',20000,'res-foreign','knowledge','doc-foreign',NULL)`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := fx.pool.Exec(ctx, statement); err != nil {
|
||||
t.Fatalf("seed target: %v\n%s", err, statement)
|
||||
}
|
||||
}
|
||||
// 目标索引自洽:目标分块必须有对应文档,否则预备份会(正确地)拒绝。
|
||||
fx.indexDoc(t, "chunk-target", wsTargetTenant, wsTargetKB, "doc-target", "图:"+wsHandleTarget)
|
||||
}
|
||||
|
||||
func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb, knowledge, content string) {
|
||||
t.Helper()
|
||||
body := fmt.Sprintf(`{"tenant_id":%d,"knowledge_base_id":%q,"knowledge_id":%q,"chunk_id":%q,"source_id":%q,"content":%q,"embedding":[0.1,0.2]}`,
|
||||
tenant, kb, knowledge, id, id, content)
|
||||
request, err := http.NewRequestWithContext(fx.ctx, "PUT", fmt.Sprintf("%s/%s/_doc/%s?refresh=true", fx.searchBase, fx.index, id),
|
||||
strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response, err := fx.client.Do(request)
|
||||
if err != nil {
|
||||
t.Fatalf("index doc: %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != 201 && response.StatusCode != 200 {
|
||||
t.Fatalf("index doc %s => %d", id, response.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。
|
||||
func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string {
|
||||
t.Helper()
|
||||
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer archive.Close()
|
||||
encode := func(value any) []byte {
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return encoded
|
||||
}
|
||||
parts := map[string][]map[string]any{
|
||||
"knowledges": {{"id": "doc-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||||
"parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}},
|
||||
"chunks": {{"id": "chunk-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||||
"knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}},
|
||||
"knowledge_tags": {},
|
||||
"knowledge_tag_relations": {},
|
||||
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": wsSourceTenant,
|
||||
"storage_backend_id": "backend-source", "provider": "oss",
|
||||
"physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-src"),
|
||||
"size": int64(8)}},
|
||||
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": wsSourceTenant,
|
||||
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}},
|
||||
"wiki_folders": {},
|
||||
"wiki_pages": {{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"}},
|
||||
"wiki_page_issues": {},
|
||||
"wiki_log_entries": {},
|
||||
}
|
||||
if tweak != nil {
|
||||
tweak(parts)
|
||||
}
|
||||
kinds := map[string]string{"knowledges": "database", "chunks": "database", "knowledge_tags": "database",
|
||||
"knowledge_tag_relations": "database", "resources": "database", "resource_bindings": "database",
|
||||
"wiki_folders": "wiki", "wiki_pages": "wiki", "wiki_page_issues": "wiki", "wiki_log_entries": "wiki"}
|
||||
manifestParts := make([]librarysnapshot.Part, 0, 14)
|
||||
names := make([]string, 0, len(parts))
|
||||
for name := range parts {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
part, err := archive.Put(bytes.NewReader(encode(parts[name])))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part.Kind, part.Name, part.Count = kinds[name], name, int64(len(parts[name]))
|
||||
manifestParts = append(manifestParts, part)
|
||||
}
|
||||
resourcePart, err := archive.Put(bytes.NewReader([]byte("12345678")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1
|
||||
manifestParts = append(manifestParts, resourcePart)
|
||||
indexPart, err := archive.Put(bytes.NewReader(encode([]map[string]any{{"id": "chunk-src", "source": map[string]any{
|
||||
"tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src",
|
||||
"chunk_id": "chunk-src", "source_id": "chunk-src", "content": "来源正文 resource://" + wsHandleSource,
|
||||
"embedding": []float64{0.1, 0.2}}}})))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", 1
|
||||
manifestParts = append(manifestParts, indexPart)
|
||||
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
|
||||
Format: librarysnapshot.Format, Scope: "internal", TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB,
|
||||
CreatedAt: time.Now().UTC(), Parts: manifestParts,
|
||||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
||||
EmbeddingDimension: 1024, IndexMappingSHA256: strings.Repeat("a", 64)}}
|
||||
id, err := archive.Seal(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("seal source snapshot: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func (fx *writeSetFixture) target(t *testing.T) *ProductionLibraryTarget {
|
||||
t.Helper()
|
||||
target, err := NewProductionLibraryTarget(fx.ctx, ProductionLibraryConfig{
|
||||
Postgres: os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), SearchURL: fx.searchBase, SearchIndex: fx.index,
|
||||
Objects: fx.objects, ObjectsReader: fx.objects, ArchiveRoot: fx.archiveRoot,
|
||||
ObjectsBucket: fx.bucket, MaxDocuments: 100, HTTPClient: fx.client,
|
||||
}, ProductionScope{TenantID: wsTargetTenant, KnowledgeBaseID: wsTargetKB})
|
||||
if err != nil {
|
||||
t.Fatalf("target: %v", err)
|
||||
}
|
||||
t.Cleanup(target.Close)
|
||||
return target
|
||||
}
|
||||
|
||||
func (fx *writeSetFixture) backups(t *testing.T, target *ProductionLibraryTarget) *ProductionBackupStore {
|
||||
t.Helper()
|
||||
mapping, err := target.Search().MappingSHA256(fx.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backups, err := NewProductionBackupStore(ProductionBackupConfig{
|
||||
Pool: target.Pool(), Search: target.Search(), Objects: fx.objects, ArchiveRoot: fx.archiveRoot,
|
||||
Scope: "internal", Identity: target.Identity(), MaxDocuments: 100,
|
||||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
||||
EmbeddingDimension: 1024, IndexMappingSHA256: mapping},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("backup store: %v", err)
|
||||
}
|
||||
return backups
|
||||
}
|
||||
|
||||
// universeDump 是"整实例逐行字节"快照:任何一行被删/被改写都会体现在这里。
|
||||
func universeDump(t *testing.T, pool *pgxpool.Pool) map[string][]string {
|
||||
t.Helper()
|
||||
dump := map[string][]string{}
|
||||
for _, table := range productionLibraryTables {
|
||||
rows, err := pool.Query(context.Background(), fmt.Sprintf(`SELECT to_jsonb(x)::text FROM %s AS x ORDER BY 1`, table))
|
||||
if err != nil {
|
||||
t.Fatalf("dump %s: %v", table, err)
|
||||
}
|
||||
values := make([]string, 0, 32)
|
||||
for rows.Next() {
|
||||
var raw string
|
||||
if err := rows.Scan(&raw); err != nil {
|
||||
rows.Close()
|
||||
t.Fatalf("dump scan %s: %v", table, err)
|
||||
}
|
||||
values = append(values, raw)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("dump %s: %v", table, err)
|
||||
}
|
||||
dump[table] = values
|
||||
}
|
||||
return dump
|
||||
}
|
||||
|
||||
func diffUniverse(before, after map[string][]string) string {
|
||||
tables := make([]string, 0, len(before))
|
||||
for table := range before {
|
||||
tables = append(tables, table)
|
||||
}
|
||||
sort.Strings(tables)
|
||||
for _, table := range tables {
|
||||
if len(before[table]) != len(after[table]) {
|
||||
return fmt.Sprintf("%s 行数 %d -> %d", table, len(before[table]), len(after[table]))
|
||||
}
|
||||
for index := range before[table] {
|
||||
if before[table][index] != after[table][index] {
|
||||
return fmt.Sprintf("%s 第 %d 行内容变了:%s", table, index, after[table][index])
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func requestFor(snapshotID string) libraryrelease.Request {
|
||||
now := time.Now().UTC()
|
||||
return libraryrelease.Request{JobID: "job-writeset-acceptance", Target: "internal", SnapshotID: snapshotID,
|
||||
ParamsDigest: "digest-writeset", Now: now,
|
||||
Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-writeset",
|
||||
ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}}
|
||||
}
|
||||
|
||||
// 场景:目标非空(目标自己的文档/分块/资源/Wiki)、同租户里还有另一个知识库的数据与资源、
|
||||
// 一份被两个知识库共享的资源、一个历史孤儿资源、另一个租户的数据;来源快照属于另一个
|
||||
// 租户/知识库;资源阶段写失败 → 必须回滚,且"目标原有数据逐行字节不变、本次残留为 0"。
|
||||
func TestLibraryRollbackKeepsPreExistingTargetDataIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
before := universeDump(t, fx.pool)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
|
||||
fx.objects.failPath = "/res-src/"
|
||||
fx.objects.failPut.Store(true)
|
||||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||||
fx.objects.failPut.Store(false)
|
||||
if outcome.State != "rolled_back" {
|
||||
t.Fatalf("outcome = %+v", outcome)
|
||||
}
|
||||
if outcome.FailedAt != libraryrelease.StageResources {
|
||||
t.Fatalf("failed_at = %s, want resources", outcome.FailedAt)
|
||||
}
|
||||
t.Logf("rollback=%s backup=%s ledger=%s", outcome.Rollback, outcome.Backup.SnapshotID,
|
||||
outcome.Backup.WriteSet.Digest)
|
||||
after := universeDump(t, fx.pool)
|
||||
if diff := diffUniverse(before, after); diff != "" {
|
||||
t.Fatalf("回滚后目标数据被改动:%s", diff)
|
||||
}
|
||||
// 历史孤儿不是本次写的:它必须还在(旧实现按"目标租户里没人引用"清理,会删掉它)。
|
||||
var orphan string
|
||||
if err := fx.pool.QueryRow(fx.ctx, `SELECT physical_path FROM resources WHERE id='res-orphan'`).Scan(&orphan); err != nil {
|
||||
t.Fatalf("历史孤儿资源被删了(恢复把'没人引用的目标租户资源'当成本次残渣):%v", err)
|
||||
}
|
||||
var sourceScope int64
|
||||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2`,
|
||||
wsSourceTenant, wsSourceKB).Scan(&sourceScope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sourceScope != 0 {
|
||||
t.Fatalf("回滚后仍有 %d 行来源范围文档", sourceScope)
|
||||
}
|
||||
var residue int64
|
||||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-src'`).Scan(&residue); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if residue != 0 {
|
||||
t.Fatalf("本次同步新增的资源行未被清理")
|
||||
}
|
||||
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
|
||||
t.Fatalf("回滚必须与预备份一致:%v", err)
|
||||
}
|
||||
// 写集合必须已落盘、可校验,并且明确区分"本次新建"与"写前已存在"。
|
||||
if outcome.Backup.WriteSet == nil || outcome.Backup.WriteSet.Digest == "" {
|
||||
t.Fatalf("预备份必须带着已落盘的写集合:%+v", outcome.Backup)
|
||||
}
|
||||
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
|
||||
if err != nil {
|
||||
t.Fatalf("写集合记录取不回来:%v", err)
|
||||
}
|
||||
createdResources := ledger.CreatedIDs("resources")
|
||||
if len(createdResources) != 1 || createdResources[0] != "res-src" {
|
||||
t.Fatalf("写集合里本次新建的资源 = %v,want [res-src]", createdResources)
|
||||
}
|
||||
// 写集合只允许包含"这次要写"的来源身份;目标自己的资源(含共享/历史孤儿/
|
||||
// 其它知识库的资源)一个都不许出现,否则恢复就会按它去删。
|
||||
created := map[string]bool{}
|
||||
for _, id := range ledger.CreatedIDs("resources") {
|
||||
created[id] = true
|
||||
}
|
||||
for _, id := range []string{"res-target", "res-shared", "res-orphan", "res-other", "res-foreign"} {
|
||||
if created[id] {
|
||||
t.Fatalf("写集合把目标原有的资源 %s 当成本次新建(恢复会删掉它):%v", id, created)
|
||||
}
|
||||
}
|
||||
if ledger.JobID != "job-writeset-acceptance" || ledger.BackupSnapshotID != outcome.Backup.SnapshotID {
|
||||
t.Fatalf("写集合没有绑定作业/预备份:%+v", ledger)
|
||||
}
|
||||
if ledger.UniverseDigest() == "" {
|
||||
t.Fatal("写集合必须带整实例内容证据")
|
||||
}
|
||||
}
|
||||
|
||||
// 场景:恢复只拿到"快照 id"(worker 重启后的真实形状)→ 没有持久化写集合就不许恢复,
|
||||
// 更不能把校验降级成"跳过整实例比对"。
|
||||
func TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
fx.objects.failPath = "/res-src/"
|
||||
fx.objects.failPut.Store(true)
|
||||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||||
fx.objects.failPut.Store(false)
|
||||
if outcome.State != "rolled_back" {
|
||||
t.Fatalf("outcome = %+v", outcome)
|
||||
}
|
||||
before := universeDump(t, fx.pool)
|
||||
// 丢掉内存里的写集合:只按快照 id 恢复(这就是 fixedBackupStore 的形状)。
|
||||
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + outcome.Backup.SnapshotID,
|
||||
SnapshotID: outcome.Backup.SnapshotID}); err == nil {
|
||||
t.Fatal("缺少持久化写集合记录时恢复必须失败(不许把校验降级成静默跳过)")
|
||||
}
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("被拒绝的恢复不许动目标:%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
// 场景:来源快照里的资源 ID 已被另一个租户占用 → 必须在写第一笔之前拒绝,
|
||||
// 不能 ON CONFLICT 改写别人的行。
|
||||
func TestLibraryPreflightRejectsCrossScopeIDConflictIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
// 另一个租户里已经有一条同 ID 的资源行。
|
||||
if _, err := fx.pool.Exec(fx.ctx, `INSERT INTO resources VALUES('res-src','`+wsHandleForeign+`',20000,'`+
|
||||
wsLocator("backend-foreign", "oss", "foreign-bucket", 20000, "res-src")+`',NULL,'oss',4,'backend-foreign')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := universeDump(t, fx.pool)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||||
if outcome.State != "failed" {
|
||||
t.Fatalf("outcome = %+v", outcome)
|
||||
}
|
||||
if outcome.FailedAt != "" {
|
||||
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at = %s", outcome.FailedAt)
|
||||
}
|
||||
if !strings.Contains(outcome.Error, "冲突") || !strings.Contains(outcome.Error, "res-src") {
|
||||
t.Fatalf("拒绝原因必须点名冲突的资源 ID:%s", outcome.Error)
|
||||
}
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("冲突预检失败后目标被改动:%s", diff)
|
||||
}
|
||||
t.Logf("conflict rejected before write: %s", outcome.Error)
|
||||
}
|
||||
|
||||
// 场景:wiki 阶段半写失败(resources 已成功上传、database 已提交)→ 回滚必须把
|
||||
// 本次新增的资源也清掉,目标原有数据逐行不变。
|
||||
func TestLibraryWikiStageHalfWriteRollbackIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
before := universeDump(t, fx.pool)
|
||||
// 同一个作用域里两页同 slug:真实 schema 的唯一索引会让 wiki 阶段写失败,
|
||||
// 而 database/resources 阶段已经成功提交(半写现场)。
|
||||
snapshotID := sealSourceSnapshot(t, fx, func(parts map[string][]map[string]any) {
|
||||
parts["wiki_pages"] = []map[string]any{
|
||||
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
|
||||
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
|
||||
}
|
||||
})
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||||
if outcome.State != "rolled_back" {
|
||||
t.Fatalf("outcome = %+v", outcome)
|
||||
}
|
||||
if outcome.FailedAt != libraryrelease.StageWiki {
|
||||
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
|
||||
}
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("wiki 阶段半写回滚后目标数据被改动:%s", diff)
|
||||
}
|
||||
if _, _, err := fx.objects.s3.Stat(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src")); err == nil {
|
||||
t.Log("note: 对象字节保留在桶里(恢复只回滚行,不删对象;对象清理不属于本次范围)")
|
||||
}
|
||||
t.Logf("wiki half-write rolled back: %s", outcome.Rollback)
|
||||
}
|
||||
|
||||
// 场景:写集合记录被删除/篡改 → 恢复必须失败并保持维护,不能"看不见记录就当没写过"。
|
||||
func TestLibraryWriteSetRecordTamperRefusesRestoreIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
fx.objects.failPath = "/res-src/"
|
||||
fx.objects.failPut.Store(true)
|
||||
outcome := libraryrelease.Run(fx.ctx, requestFor(snapshotID), target, backups, nil)
|
||||
fx.objects.failPut.Store(false)
|
||||
if outcome.State != "rolled_back" || outcome.Backup.WriteSet == nil {
|
||||
t.Fatalf("outcome = %+v", outcome)
|
||||
}
|
||||
digest := outcome.Backup.WriteSet.Digest
|
||||
path := writeSetPath(fx.archiveRoot, digest)
|
||||
|
||||
// 1) 记录被篡改:同一个文件改一个字段 → 取回必须失败(摘要对不上)。
|
||||
original, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tampered := bytes.Replace(original, []byte(`"tenant_id":10001`), []byte(`"tenant_id":10002`), 1)
|
||||
if bytes.Equal(tampered, original) {
|
||||
t.Fatalf("篡改夹具没生效:%s", original)
|
||||
}
|
||||
if err := os.Chmod(path, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, tampered, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
|
||||
t.Fatal("写集合被篡改后取回必须失败")
|
||||
}
|
||||
// 内存里带着被篡改的记录去恢复,同样必须失败。
|
||||
bad := *outcome.Backup.WriteSet
|
||||
bad.TenantID = 10002
|
||||
if err := target.Restore(fx.ctx, libraryrelease.BackupRef{ID: outcome.Backup.ID,
|
||||
SnapshotID: outcome.Backup.SnapshotID, WriteSet: &bad}); err == nil {
|
||||
t.Fatal("写集合内容与摘要不符时恢复必须失败")
|
||||
}
|
||||
|
||||
// 2) 记录被删除:恢复必须失败并保持维护(不是静默跳过校验)。
|
||||
if err := os.WriteFile(path, original, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadWriteSet(fx.archiveRoot, digest); err == nil {
|
||||
t.Fatal("写集合记录被删除后取回必须失败")
|
||||
}
|
||||
t.Logf("tampered/removed ledger refused, digest=%s", digest)
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// 真杀进程的接管验证:子进程在 resources 阶段被 SIGKILL(模拟 worker 被杀),
|
||||
// 已经提交的 database 阶段与"写集合 + 预备份"记录留在盘上;父进程作为"重启后的
|
||||
// worker"只按持久记录恢复。
|
||||
//
|
||||
// 说明边界:杀的是**发布路径的进程**(本文件的 helper),不是治理栈里的 worker 容器;
|
||||
// 队列/UI 侧的接管由父负责页面验收。这里证明的是"进程没了之后,恢复只用盘上的
|
||||
// 写集合与整实例证据,并且删不到目标原有数据"。
|
||||
|
||||
const (
|
||||
killChildEnv = "KG_WRITESET_KILL_CHILD"
|
||||
killStateEnv = "KG_WRITESET_KILL_STATE"
|
||||
killJournalEnv = "KG_WRITESET_KILL_JOURNAL"
|
||||
)
|
||||
|
||||
// killHelperState 是子进程写下的"治理库记录"替身:真实系统里这两项分别落在
|
||||
// library_jobs.snapshot_id 与 library_versions.evidence.writeset_digest。
|
||||
type killHelperState struct {
|
||||
JobID string `json:"job_id"`
|
||||
BackupSnapshotID string `json:"backup_snapshot_id"`
|
||||
TargetSnapshotID string `json:"target_snapshot_id"`
|
||||
WriteSetDigest string `json:"writeset_digest"`
|
||||
}
|
||||
|
||||
func TestWriteSetKillHelper(t *testing.T) {
|
||||
if os.Getenv(killChildEnv) != "1" {
|
||||
t.Skip("helper process only")
|
||||
}
|
||||
// 子进程复用父进程的现场:不重建 schema/索引,写集合与预备份仍落在同一个归档根。
|
||||
fx := openWriteSetFixtureWith(t, writeSetFixtureOptions{archiveRoot: os.Getenv(killJournalEnv + "_ARCHIVE"),
|
||||
index: os.Getenv(killJournalEnv + "_INDEX")})
|
||||
state := os.Getenv(killStateEnv)
|
||||
journal := os.Getenv(killJournalEnv)
|
||||
snapshotID := os.Getenv("KG_WRITESET_KILL_SNAPSHOT")
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
current, err := target.Current(fx.ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("current: %v", err)
|
||||
}
|
||||
backup, err := backups.Prepare(fx.ctx, snapshotID, current)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare backup: %v", err)
|
||||
}
|
||||
set, err := target.PlanWriteSet(fx.ctx, libraryrelease.WriteSetPlan{JobID: "job-kill", Target: "internal",
|
||||
BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: snapshotID, Universe: backup.Universe})
|
||||
if err != nil {
|
||||
t.Fatalf("plan write set: %v", err)
|
||||
}
|
||||
record, err := json.Marshal(killHelperState{JobID: set.JobID, BackupSnapshotID: backup.SnapshotID,
|
||||
TargetSnapshotID: snapshotID, WriteSetDigest: set.Digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(state, record, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(journal, []byte(fx.archiveRoot), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := target.Apply(fx.ctx, libraryrelease.StageDatabase, snapshotID); err != nil {
|
||||
t.Fatalf("apply database: %v", err)
|
||||
}
|
||||
// resources 阶段:第一笔上传就把自己杀掉(真 SIGKILL,没有 defer、没有收尾)。
|
||||
dying := &killOnPutStore{inner: fx.objects.s3}
|
||||
target.config.Objects = dying
|
||||
if err := target.Apply(fx.ctx, libraryrelease.StageResources, snapshotID); err != nil {
|
||||
t.Fatalf("apply resources: %v", err)
|
||||
}
|
||||
t.Fatal("helper should have been killed during the resources stage")
|
||||
}
|
||||
|
||||
// killOnPutStore 在第一次上传时直接 SIGKILL 自己:模拟进程被外部杀掉。
|
||||
type killOnPutStore struct {
|
||||
inner *S3ObjectStore
|
||||
}
|
||||
|
||||
func (s *killOnPutStore) Put(_ context.Context, _, _ string, _ io.Reader, _ int64) error {
|
||||
if err := syscall.Kill(os.Getpid(), syscall.SIGKILL); err != nil {
|
||||
return err
|
||||
}
|
||||
select {}
|
||||
}
|
||||
|
||||
func (s *killOnPutStore) Stat(ctx context.Context, provider, path string) (int64, string, error) {
|
||||
return s.inner.Stat(ctx, provider, path)
|
||||
}
|
||||
|
||||
func (s *killOnPutStore) Copy(ctx context.Context, provider, source, target string) error {
|
||||
return s.inner.Copy(ctx, provider, source, target)
|
||||
}
|
||||
|
||||
func TestLibraryKillWorkerThenResumeRecoversWriteSetIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
before := universeDump(t, fx.pool)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
|
||||
statePath := t.TempDir() + "/kill.json"
|
||||
journalPath := t.TempDir() + "/journal.txt"
|
||||
command := exec.Command(os.Args[0], "-test.run=TestWriteSetKillHelper", "-test.v")
|
||||
command.Env = append(os.Environ(), killChildEnv+"=1", killStateEnv+"="+statePath,
|
||||
killJournalEnv+"="+journalPath, killJournalEnv+"_ARCHIVE="+fx.archiveRoot,
|
||||
killJournalEnv+"_INDEX="+fx.index, "KG_WRITESET_KILL_SNAPSHOT="+snapshotID)
|
||||
output, err := command.CombinedOutput()
|
||||
if err == nil {
|
||||
t.Fatalf("helper must be killed, it exited cleanly: %s", output)
|
||||
}
|
||||
exitErr, ok := err.(*exec.ExitError)
|
||||
if !ok {
|
||||
t.Fatalf("helper failed to run: %v\n%s", err, output)
|
||||
}
|
||||
status, ok := exitErr.Sys().(syscall.WaitStatus)
|
||||
if !ok || !status.Signaled() || status.Signal() != syscall.SIGKILL {
|
||||
t.Fatalf("helper must die from SIGKILL, got %v\n%s", err, output)
|
||||
}
|
||||
t.Logf("helper killed by %s (half-written target)", status.Signal())
|
||||
|
||||
raw, err := os.ReadFile(statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("child did not persist the job record: %v", err)
|
||||
}
|
||||
var record killHelperState
|
||||
if err := json.Unmarshal(raw, &record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.WriteSetDigest == "" || record.BackupSnapshotID == "" {
|
||||
t.Fatalf("child must persist writeset digest + backup snapshot: %+v", record)
|
||||
}
|
||||
journalRoot, err := os.ReadFile(journalPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.TrimSpace(string(journalRoot)) != fx.archiveRoot {
|
||||
t.Fatalf("journal root mismatch: %q", journalRoot)
|
||||
}
|
||||
|
||||
// 重启后的 worker:新连接、新目标对象,只拿盘上的记录。
|
||||
resumedTarget := fx.target(t)
|
||||
ledger, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest)
|
||||
if err != nil {
|
||||
t.Fatalf("重启后取回写集合失败:%v", err)
|
||||
}
|
||||
if ledger.JobID != record.JobID || ledger.BackupSnapshotID != record.BackupSnapshotID ||
|
||||
ledger.TargetSnapshotID != record.TargetSnapshotID {
|
||||
t.Fatalf("写集合与作业记录不一致:%+v vs %+v", ledger, record)
|
||||
}
|
||||
if err := resumedTarget.Restore(fx.ctx, libraryrelease.BackupRef{ID: "backup-" + record.BackupSnapshotID,
|
||||
SnapshotID: record.BackupSnapshotID, WriteSet: ledger}); err != nil {
|
||||
t.Fatalf("接管恢复失败:%v", err)
|
||||
}
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("接管恢复后目标数据被改动:%s", diff)
|
||||
}
|
||||
if err := resumedTarget.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: record.BackupSnapshotID}); err != nil {
|
||||
t.Fatalf("接管恢复后必须与预备份一致:%v", err)
|
||||
}
|
||||
t.Logf("resumed after SIGKILL: job=%s backup=%s ledger=%s", record.JobID, record.BackupSnapshotID, record.WriteSetDigest)
|
||||
|
||||
// 负例:记录被移除 → 恢复必须失败(维护保持),而不是"看不见就当没写过"。
|
||||
if err := os.Chmod(writeSetPath(fx.archiveRoot, record.WriteSetDigest), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(writeSetPath(fx.archiveRoot, record.WriteSetDigest)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadWriteSet(fx.archiveRoot, record.WriteSetDigest); err == nil {
|
||||
t.Fatal("写集合被移除后必须取不回来")
|
||||
}
|
||||
_ = time.Now()
|
||||
}
|
||||
@@ -187,6 +187,41 @@
|
||||
"package": "knowledge-governance/internal/qawemail",
|
||||
"test": "TestScheduledSendRequiresConfirmationAndSendsExactlyOnce",
|
||||
"reason": "需要一个一次性 PostgreSQL 与测试主密钥(KG_WEEKLY_MAIL_TEST_DSN/KG_WEEKLY_MAIL_TEST_KEY),CI 不提供;本批已在隔离库真实跑通并通过"
|
||||
},
|
||||
{
|
||||
"package": "knowledge-governance/internal/releaseadapter/v071",
|
||||
"test": "TestLibraryRollbackKeepsPreExistingTargetDataIsolated",
|
||||
"reason": "需要一次性隔离库 kg_library_test_*、随机 ES 索引与真实对象存储(KG_LIBRARY_TEST_S3_*);验证目标非空/同租户其它知识库/共享资源/历史孤儿/跨租户数据在失败恢复后逐行字节不变,默认 CI 无这些服务,本机已实跑通过"
|
||||
},
|
||||
{
|
||||
"package": "knowledge-governance/internal/releaseadapter/v071",
|
||||
"test": "TestLibraryRestoreRefusesWithoutWriteSetRecordIsolated",
|
||||
"reason": "同上环境;验证缺少持久化写集合时恢复必须失败且不动目标,默认 CI 跳过不算验证"
|
||||
},
|
||||
{
|
||||
"package": "knowledge-governance/internal/releaseadapter/v071",
|
||||
"test": "TestLibraryPreflightRejectsCrossScopeIDConflictIsolated",
|
||||
"reason": "同上环境;验证跨范围同 ID 资源冲突在任何写入之前被拒绝,默认 CI 跳过不算验证"
|
||||
},
|
||||
{
|
||||
"package": "knowledge-governance/internal/releaseadapter/v071",
|
||||
"test": "TestLibraryWikiStageHalfWriteRollbackIsolated",
|
||||
"reason": "同上环境;验证 wiki 阶段半写(database/resources 已提交)后回滚不留残渣,默认 CI 跳过不算验证"
|
||||
},
|
||||
{
|
||||
"package": "knowledge-governance/internal/releaseadapter/v071",
|
||||
"test": "TestLibraryWriteSetRecordTamperRefusesRestoreIsolated",
|
||||
"reason": "同上环境;验证写集合记录被篡改/删除后取回与恢复都失败,默认 CI 跳过不算验证"
|
||||
},
|
||||
{
|
||||
"package": "knowledge-governance/internal/releaseadapter/v071",
|
||||
"test": "TestLibraryKillWorkerThenResumeRecoversWriteSetIsolated",
|
||||
"reason": "同上环境并用 SIGKILL 杀掉子进程模拟 worker 被杀,验证重启后只按持久写集合恢复且目标原有数据不变;需要本地真实 PG/ES/MinIO,默认 CI 跳过不算验证"
|
||||
},
|
||||
{
|
||||
"package": "knowledge-governance/internal/releaseadapter/v071",
|
||||
"test": "TestWriteSetKillHelper",
|
||||
"reason": "上面 SIGKILL 用例的子进程 fixture,单独运行没有意义;默认 CI 跳过"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user