0252492e53
同步 A 成功后手工恢复 B,resources 表既不删行也不删对象;再同步 A 时, resources 里那 21 行以“同租户但不在预备份范围”被判成别人的知识库,整次 同步在写第一笔之前被拒。 “目标里没人引用”不是归属证据:历史孤儿、其它知识库暂未绑定的资源都满足 它。改成唯一放行路径——可审计复用(v071/library_reuse.go): 1. 持久来源:这个目标成功同步过且登记过生效版本的快照,或它自己的预备份, 且同一资源行在来源里内容一致; 2. 整行内容(除易变时间戳)与本次写入将要落地的行完全一致; 3. 该资源的对象字节与本次快照归档摘要一致。 三条同时成立才允许复用:写入阶段跳过它(不重写行、不重传对象),写集合里 记 existing+reused,恢复不清理;证据单独落盘 reuse/<digest>.json,摘要写进 写集合行,不冒充历史台账。内容不同、来源范围不同、或没有持久来源一律继续 拒绝,且拒绝发生在写入之前。 为什么把预备份也算来源:恢复作业的写集合同样会把“本目标历史状态里已有的 资源”判成跨范围冲突,而它们本来就出自这个目标自己的快照。 验证:隔离 PG+ES+MinIO 真服务下新增 5 个场景用例(复用成功且 xmin 不变、 无来源拒绝、内容不同拒绝、同内容不同来源拒绝、wiki 半写回滚只清本次新建); v071 整包与全仓 go test 通过;make verify 15 项全绿。
306 lines
14 KiB
Go
306 lines
14 KiB
Go
package v071
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"knowledge-governance/internal/libraryrelease"
|
||
)
|
||
|
||
// 这一组用例覆盖真实缺陷现场:同步 A 成功 → 手工恢复 B(对象保留、资源行不删)
|
||
// → 再同步 A。旧预检把"恢复保留下来的、本目标自己上次写进去的资源"判成
|
||
// "其它知识库的同 ID 冲突",整个同步在写第一笔之前被拒。
|
||
//
|
||
// 复用的放行条件必须是可审计的证据,而不是"目标里没人引用":
|
||
// - 目标现状那一行与本次要写的行逐列相同(除易变时间戳);
|
||
// - 该资源的对象字节与本次快照归档摘要一致;
|
||
// - 治理库里存在"这个目标成功同步过、且含有同一行内容"的快照/作业。
|
||
// 缺任何一条都必须继续拒绝,且拒绝发生在写入之前。
|
||
|
||
const (
|
||
wsHandleNew = "gggggggggggggggggggggg"
|
||
// resNewBody 是"A 的下一次发布"新增资源的内容;与 res-src 的 "12345678" 不同,
|
||
// 这样"本次真新建"和"证明可复用"在同一份写集合里能同时被观察到。
|
||
resNewBody = "87654321"
|
||
)
|
||
|
||
// reusePriors 是治理库会给出的来源清单:一次成功同步 + 它产生的预备份。
|
||
func reusePriors(syncJobID, syncSnapshotID string, prebackups ...string) []libraryrelease.PriorSnapshot {
|
||
priors := []libraryrelease.PriorSnapshot{{JobID: syncJobID, SnapshotID: syncSnapshotID, Kind: libraryrelease.PriorSnapshotSync}}
|
||
for _, snapshot := range prebackups {
|
||
priors = append(priors, libraryrelease.PriorSnapshot{JobID: syncJobID, SnapshotID: snapshot,
|
||
Kind: libraryrelease.PriorSnapshotPrebackup})
|
||
}
|
||
return priors
|
||
}
|
||
|
||
func requestForWithPrior(snapshotID, jobID string, prior []libraryrelease.PriorSnapshot) libraryrelease.Request {
|
||
request := requestFor(snapshotID)
|
||
request.JobID = jobID
|
||
request.PriorSnapshots = prior
|
||
return request
|
||
}
|
||
|
||
// syncThenRestore 造出现场:同步 A 成功,然后按生产恢复作业的方式恢复到 A 的
|
||
// 预备份(再走一次 Run:新的预备份 + 它自己的写集合)。恢复不删 resources 表,
|
||
// 所以 A 写进去的资源会作为"没人引用的行"留在目标里——正是本次缺陷的前提。
|
||
func syncThenRestore(t *testing.T, fx *writeSetFixture, snapshotID string) (*ProductionLibraryTarget,
|
||
*ProductionBackupStore, libraryrelease.Outcome, []libraryrelease.PriorSnapshot) {
|
||
t.Helper()
|
||
target := fx.target(t)
|
||
backups := fx.backups(t, target)
|
||
first := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a", nil), target, backups, nil)
|
||
if first.State != "succeeded" {
|
||
t.Fatalf("首次同步 A 必须成功:%+v", first)
|
||
}
|
||
if first.Backup.WriteSet == nil {
|
||
t.Fatal("首次同步必须落盘写集合")
|
||
}
|
||
// 恢复作业同样只认已登记来源:它自己刚拍的那份预备份就是本目标的历史状态。
|
||
restore := libraryrelease.Run(fx.ctx, requestForWithPrior(first.Backup.SnapshotID, "job-restore-b",
|
||
reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)), target, backups, nil)
|
||
if restore.State != "succeeded" {
|
||
t.Fatalf("手工恢复 B 必须成功:%+v", restore)
|
||
}
|
||
var bindings int64
|
||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resource_bindings WHERE resource_id='res-src'`).Scan(&bindings); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if bindings != 0 {
|
||
t.Fatalf("恢复后 res-src 不应还有绑定行(现场前提):%d", bindings)
|
||
}
|
||
return target, backups, first, reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)
|
||
}
|
||
|
||
// resourceFingerprint 取 PG 行的 xmin 与整行 JSON:xmin 不变才能证明"没有重写这一行"。
|
||
func resourceFingerprint(t *testing.T, pool *pgxpool.Pool, id string) (string, string) {
|
||
t.Helper()
|
||
var xmin, row string
|
||
if err := pool.QueryRow(t.Context(), `SELECT xmin::text, to_jsonb(r)::text FROM resources r WHERE id=$1`, id).
|
||
Scan(&xmin, &row); err != nil {
|
||
t.Fatalf("读取资源 %s: %v", id, err)
|
||
}
|
||
return xmin, row
|
||
}
|
||
|
||
func resourceRowOf(t *testing.T, pool *pgxpool.Pool, id string) string {
|
||
t.Helper()
|
||
var row string
|
||
if err := pool.QueryRow(t.Context(), `SELECT to_jsonb(r)::text FROM resources r WHERE id=$1`, id).Scan(&row); err != nil {
|
||
t.Fatalf("读取资源 %s: %v", id, err)
|
||
}
|
||
return row
|
||
}
|
||
|
||
func TestLibraryResyncAfterRestoreReusesProvenResourcesIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target, backups, first, priors := syncThenRestore(t, fx, snapshotID)
|
||
|
||
beforeXmin, beforeRow := resourceFingerprint(t, fx.pool, "res-src")
|
||
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
|
||
otherBefore := resourceRowOf(t, fx.pool, "res-other")
|
||
sharedBefore := resourceRowOf(t, fx.pool, "res-shared")
|
||
foreignBefore := resourceRowOf(t, fx.pool, "res-foreign")
|
||
|
||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a-again", priors),
|
||
target, backups, nil)
|
||
if outcome.State != "succeeded" {
|
||
t.Fatalf("恢复后再次同步 A 必须完成:%+v", outcome)
|
||
}
|
||
if outcome.Backup.WriteSet == nil {
|
||
t.Fatal("写集合必须落盘")
|
||
}
|
||
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
|
||
if err != nil {
|
||
t.Fatalf("写集合记录取不回来:%v", err)
|
||
}
|
||
reused := ledger.ReusedIDs("resources")
|
||
if len(reused) != 1 || reused[0] != "res-src" {
|
||
t.Fatalf("复用资源 = %v,want [res-src]", reused)
|
||
}
|
||
if created := ledger.CreatedIDs("resources"); len(created) != 0 {
|
||
t.Fatalf("复用资源不能被记成本次新建(恢复会删掉它):%v", created)
|
||
}
|
||
var reuseRow libraryrelease.WriteSetRow
|
||
for _, row := range ledger.Rows {
|
||
if row.Table == "resources" && row.ID == "res-src" {
|
||
reuseRow = row
|
||
}
|
||
}
|
||
if !reuseRow.Existing || !reuseRow.Reused || reuseRow.ReuseEvidence == "" {
|
||
t.Fatalf("复用行必须带证据摘要并标记为写前已存在:%+v", reuseRow)
|
||
}
|
||
evidence, err := LoadReuseEvidence(fx.archiveRoot, reuseRow.ReuseEvidence)
|
||
if err != nil {
|
||
t.Fatalf("复用证据取不回来:%v", err)
|
||
}
|
||
if evidence.ProvenanceSnapshotID != snapshotID || evidence.ProvenanceJobID != first.Backup.WriteSet.JobID {
|
||
t.Fatalf("复用证据没有指向前一次成功同步:%+v", evidence)
|
||
}
|
||
if evidence.TargetRowDigest != evidence.RetargetedRowDigest || evidence.ProvenanceRowDigest != evidence.ArchivedRowDigest {
|
||
t.Fatalf("复用证据的行摘要不一致:%+v", evidence)
|
||
}
|
||
if len(evidence.Objects) != 1 || evidence.Objects[0].SHA256 == "" {
|
||
t.Fatalf("复用证据必须逐个记录对象字节:%+v", evidence.Objects)
|
||
}
|
||
|
||
// 复用不许改原行:xmin 与整行 JSON 都必须原样。
|
||
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
|
||
if afterXmin != beforeXmin {
|
||
t.Fatalf("复用资源被重写了:xmin %s -> %s", beforeXmin, afterXmin)
|
||
}
|
||
if afterRow != beforeRow {
|
||
t.Fatalf("复用资源内容变了:\n%s\n->\n%s", beforeRow, afterRow)
|
||
}
|
||
// 历史孤儿/其它知识库/共享/外租户资源一律不许被覆盖。
|
||
for id, want := range map[string]string{"res-orphan": orphanBefore, "res-other": otherBefore,
|
||
"res-shared": sharedBefore, "res-foreign": foreignBefore} {
|
||
if got := resourceRowOf(t, fx.pool, id); got != want {
|
||
t.Fatalf("资源 %s 被改动:\n%s\n->\n%s", id, want, got)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestLibraryResyncAfterRestoreRejectsReuseWithoutProvenanceIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target, backups, first, _ := syncThenRestore(t, fx, snapshotID)
|
||
before := universeDump(t, fx.pool)
|
||
|
||
// 已登记的来源里没有 res-src(只有这次同步之前的预备份):即使目标那一行
|
||
// 看起来一模一样,也不许放行。
|
||
noSource := []libraryrelease.PriorSnapshot{{JobID: first.Backup.WriteSet.JobID,
|
||
SnapshotID: first.Backup.SnapshotID, Kind: libraryrelease.PriorSnapshotPrebackup}}
|
||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-no-provenance", noSource), target, backups, nil)
|
||
if outcome.State != "failed" {
|
||
t.Fatalf("没有来源证据时必须拒绝:%+v", outcome)
|
||
}
|
||
if outcome.FailedAt != "" {
|
||
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at=%s", outcome.FailedAt)
|
||
}
|
||
if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "可审计") {
|
||
t.Fatalf("拒绝原因必须点名资源并说明缺来源证据:%s", outcome.Error)
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
|
||
}
|
||
}
|
||
|
||
func TestLibraryResyncAfterRestoreRejectsReuseWhenContentDiffersIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
|
||
// 同 ID 同租户,但内容被改过(这里改大小,等价于"另一份文件占了这个 ID")。
|
||
if _, err := fx.pool.Exec(fx.ctx, `UPDATE resources SET size=size+1 WHERE id='res-src'`); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
changed := resourceRowOf(t, fx.pool, "res-src")
|
||
before := universeDump(t, fx.pool)
|
||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-content-diff", priors), target, backups, nil)
|
||
if outcome.State != "failed" {
|
||
t.Fatalf("内容不一致必须拒绝:%+v", outcome)
|
||
}
|
||
if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "内容") {
|
||
t.Fatalf("拒绝原因必须点名内容不一致:%s", outcome.Error)
|
||
}
|
||
if resourceRowOf(t, fx.pool, "res-src") != changed {
|
||
t.Fatal("被拒绝的同步不许改目标资源")
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
|
||
}
|
||
}
|
||
|
||
// 同租户、同 ID、同内容,但来源快照属于另一个知识库:仍然不许复用。
|
||
// "归属无法证明"包括"来源不同",不能因为字节一样就把别人的行当成自己的。
|
||
func TestLibraryReuseRequiresSameSourceScopeIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
|
||
otherSource := sealSourceSnapshotScope(t, fx, wsSourceTenant+1, "kb-source-other", nil, nil)
|
||
before := universeDump(t, fx.pool)
|
||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(otherSource, "job-sync-other-source", priors), target, backups, nil)
|
||
if outcome.State != "failed" {
|
||
t.Fatalf("来源范围不同必须拒绝复用:%+v", outcome)
|
||
}
|
||
if !strings.Contains(outcome.Error, "可审计") {
|
||
t.Fatalf("拒绝原因必须说明来源不可审计:%s", outcome.Error)
|
||
}
|
||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
|
||
}
|
||
}
|
||
|
||
// 部分写失败:database/resources 已提交、wiki 因重复 slug 失败 → 回滚只清"本次真新建",
|
||
// 复用资源与历史孤儿原内容完整保留(xmin 也不变)。
|
||
func TestLibraryHalfWriteRollbackKeepsReusedResourcesIsolated(t *testing.T) {
|
||
fx := openWriteSetFixture(t)
|
||
seedTargetData(t, fx)
|
||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
|
||
reuseXmin, reuseRow := resourceFingerprint(t, fx.pool, "res-src")
|
||
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
|
||
|
||
next := sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB,
|
||
func(parts map[string][]map[string]any) {
|
||
parts["chunks"] = append(parts["chunks"], map[string]any{"id": "chunk-src-2", "tenant_id": wsSourceTenant,
|
||
"knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src", "content": "新增正文 resource://" + wsHandleNew})
|
||
parts["resources"] = append(parts["resources"], map[string]any{"id": "res-new", "handle": wsHandleNew,
|
||
"tenant_id": wsSourceTenant, "storage_backend_id": "backend-source", "provider": "oss",
|
||
"physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-new"),
|
||
"size": int64(len(resNewBody))})
|
||
parts["resource_bindings"] = append(parts["resource_bindings"], map[string]any{"id": "bind-src-2",
|
||
"resource_id": "res-new", "tenant_id": wsSourceTenant, "owner_type": "chunk",
|
||
"owner_id": "chunk-src-2", "relation": "attachment"})
|
||
// 同一作用域两页同 slug:wiki 的唯一索引让 wiki 阶段失败(真实半写现场)。
|
||
parts["wiki_pages"] = []map[string]any{
|
||
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
|
||
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
|
||
}
|
||
}, map[string]string{"res-new": resNewBody})
|
||
|
||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(next, "job-sync-half-write", priors), target, backups, nil)
|
||
if outcome.State != "rolled_back" {
|
||
t.Fatalf("wiki 半写必须回滚:%+v", outcome)
|
||
}
|
||
if outcome.FailedAt != libraryrelease.StageWiki {
|
||
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
|
||
}
|
||
ledger := outcome.Backup.WriteSet
|
||
if ledger == nil {
|
||
t.Fatal("半写作业必须留下写集合")
|
||
}
|
||
if created := ledger.CreatedIDs("resources"); len(created) != 1 || created[0] != "res-new" {
|
||
t.Fatalf("本次真新建的资源 = %v,want [res-new]", created)
|
||
}
|
||
if reused := ledger.ReusedIDs("resources"); len(reused) != 1 || reused[0] != "res-src" {
|
||
t.Fatalf("复用资源 = %v,want [res-src]", reused)
|
||
}
|
||
// 本次真新建的清干净。
|
||
var residue int64
|
||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-new'`).Scan(&residue); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if residue != 0 {
|
||
t.Fatal("回滚后本次新建的资源必须被清理")
|
||
}
|
||
// 复用资源与历史孤儿原内容(含 xmin)保留。
|
||
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
|
||
if afterXmin != reuseXmin || afterRow != reuseRow {
|
||
t.Fatalf("回滚动了复用资源:xmin %s->%s\n%s\n->\n%s", reuseXmin, afterXmin, reuseRow, afterRow)
|
||
}
|
||
if resourceRowOf(t, fx.pool, "res-orphan") != orphanBefore {
|
||
t.Fatal("回滚动了历史孤儿资源")
|
||
}
|
||
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
|
||
t.Fatalf("回滚必须与预备份一致:%v", err)
|
||
}
|
||
}
|