Files
knowledge-governance/internal/releaseadapter/v071/library_reuse_integration_test.go
T
姚章浩 0252492e53 fix(library): 恢复保留下来的资源凭可审计证据复用
同步 A 成功后手工恢复 B,resources 表既不删行也不删对象;再同步 A 时,
resources 里那 21 行以“同租户但不在预备份范围”被判成别人的知识库,整次
同步在写第一笔之前被拒。

“目标里没人引用”不是归属证据:历史孤儿、其它知识库暂未绑定的资源都满足
它。改成唯一放行路径——可审计复用(v071/library_reuse.go):
1. 持久来源:这个目标成功同步过且登记过生效版本的快照,或它自己的预备份,
   且同一资源行在来源里内容一致;
2. 整行内容(除易变时间戳)与本次写入将要落地的行完全一致;
3. 该资源的对象字节与本次快照归档摘要一致。

三条同时成立才允许复用:写入阶段跳过它(不重写行、不重传对象),写集合里
记 existing+reused,恢复不清理;证据单独落盘 reuse/<digest>.json,摘要写进
写集合行,不冒充历史台账。内容不同、来源范围不同、或没有持久来源一律继续
拒绝,且拒绝发生在写入之前。

为什么把预备份也算来源:恢复作业的写集合同样会把“本目标历史状态里已有的
资源”判成跨范围冲突,而它们本来就出自这个目标自己的快照。

验证:隔离 PG+ES+MinIO 真服务下新增 5 个场景用例(复用成功且 xmin 不变、
无来源拒绝、内容不同拒绝、同内容不同来源拒绝、wiki 半写回滚只清本次新建);
v071 整包与全仓 go test 通过;make verify 15 项全绿。
2026-10-30 01:59:32 +08:00

306 lines
14 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryrelease"
)
// 这一组用例覆盖真实缺陷现场:同步 A 成功 → 手工恢复 B(对象保留、资源行不删)
// → 再同步 A。旧预检把"恢复保留下来的、本目标自己上次写进去的资源"判成
// "其它知识库的同 ID 冲突",整个同步在写第一笔之前被拒。
//
// 复用的放行条件必须是可审计的证据,而不是"目标里没人引用":
// - 目标现状那一行与本次要写的行逐列相同(除易变时间戳);
// - 该资源的对象字节与本次快照归档摘要一致;
// - 治理库里存在"这个目标成功同步过、且含有同一行内容"的快照/作业。
// 缺任何一条都必须继续拒绝,且拒绝发生在写入之前。
const (
wsHandleNew = "gggggggggggggggggggggg"
// resNewBody 是"A 的下一次发布"新增资源的内容;与 res-src 的 "12345678" 不同,
// 这样"本次真新建"和"证明可复用"在同一份写集合里能同时被观察到。
resNewBody = "87654321"
)
// reusePriors 是治理库会给出的来源清单:一次成功同步 + 它产生的预备份。
func reusePriors(syncJobID, syncSnapshotID string, prebackups ...string) []libraryrelease.PriorSnapshot {
priors := []libraryrelease.PriorSnapshot{{JobID: syncJobID, SnapshotID: syncSnapshotID, Kind: libraryrelease.PriorSnapshotSync}}
for _, snapshot := range prebackups {
priors = append(priors, libraryrelease.PriorSnapshot{JobID: syncJobID, SnapshotID: snapshot,
Kind: libraryrelease.PriorSnapshotPrebackup})
}
return priors
}
func requestForWithPrior(snapshotID, jobID string, prior []libraryrelease.PriorSnapshot) libraryrelease.Request {
request := requestFor(snapshotID)
request.JobID = jobID
request.PriorSnapshots = prior
return request
}
// syncThenRestore 造出现场:同步 A 成功,然后按生产恢复作业的方式恢复到 A 的
// 预备份(再走一次 Run:新的预备份 + 它自己的写集合)。恢复不删 resources 表,
// 所以 A 写进去的资源会作为"没人引用的行"留在目标里——正是本次缺陷的前提。
func syncThenRestore(t *testing.T, fx *writeSetFixture, snapshotID string) (*ProductionLibraryTarget,
*ProductionBackupStore, libraryrelease.Outcome, []libraryrelease.PriorSnapshot) {
t.Helper()
target := fx.target(t)
backups := fx.backups(t, target)
first := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a", nil), target, backups, nil)
if first.State != "succeeded" {
t.Fatalf("首次同步 A 必须成功:%+v", first)
}
if first.Backup.WriteSet == nil {
t.Fatal("首次同步必须落盘写集合")
}
// 恢复作业同样只认已登记来源:它自己刚拍的那份预备份就是本目标的历史状态。
restore := libraryrelease.Run(fx.ctx, requestForWithPrior(first.Backup.SnapshotID, "job-restore-b",
reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)), target, backups, nil)
if restore.State != "succeeded" {
t.Fatalf("手工恢复 B 必须成功:%+v", restore)
}
var bindings int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resource_bindings WHERE resource_id='res-src'`).Scan(&bindings); err != nil {
t.Fatal(err)
}
if bindings != 0 {
t.Fatalf("恢复后 res-src 不应还有绑定行(现场前提):%d", bindings)
}
return target, backups, first, reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)
}
// resourceFingerprint 取 PG 行的 xmin 与整行 JSON:xmin 不变才能证明"没有重写这一行"。
func resourceFingerprint(t *testing.T, pool *pgxpool.Pool, id string) (string, string) {
t.Helper()
var xmin, row string
if err := pool.QueryRow(t.Context(), `SELECT xmin::text, to_jsonb(r)::text FROM resources r WHERE id=$1`, id).
Scan(&xmin, &row); err != nil {
t.Fatalf("读取资源 %s: %v", id, err)
}
return xmin, row
}
func resourceRowOf(t *testing.T, pool *pgxpool.Pool, id string) string {
t.Helper()
var row string
if err := pool.QueryRow(t.Context(), `SELECT to_jsonb(r)::text FROM resources r WHERE id=$1`, id).Scan(&row); err != nil {
t.Fatalf("读取资源 %s: %v", id, err)
}
return row
}
func TestLibraryResyncAfterRestoreReusesProvenResourcesIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, first, priors := syncThenRestore(t, fx, snapshotID)
beforeXmin, beforeRow := resourceFingerprint(t, fx.pool, "res-src")
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
otherBefore := resourceRowOf(t, fx.pool, "res-other")
sharedBefore := resourceRowOf(t, fx.pool, "res-shared")
foreignBefore := resourceRowOf(t, fx.pool, "res-foreign")
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a-again", priors),
target, backups, nil)
if outcome.State != "succeeded" {
t.Fatalf("恢复后再次同步 A 必须完成:%+v", outcome)
}
if outcome.Backup.WriteSet == nil {
t.Fatal("写集合必须落盘")
}
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
if err != nil {
t.Fatalf("写集合记录取不回来:%v", err)
}
reused := ledger.ReusedIDs("resources")
if len(reused) != 1 || reused[0] != "res-src" {
t.Fatalf("复用资源 = %v,want [res-src]", reused)
}
if created := ledger.CreatedIDs("resources"); len(created) != 0 {
t.Fatalf("复用资源不能被记成本次新建(恢复会删掉它):%v", created)
}
var reuseRow libraryrelease.WriteSetRow
for _, row := range ledger.Rows {
if row.Table == "resources" && row.ID == "res-src" {
reuseRow = row
}
}
if !reuseRow.Existing || !reuseRow.Reused || reuseRow.ReuseEvidence == "" {
t.Fatalf("复用行必须带证据摘要并标记为写前已存在:%+v", reuseRow)
}
evidence, err := LoadReuseEvidence(fx.archiveRoot, reuseRow.ReuseEvidence)
if err != nil {
t.Fatalf("复用证据取不回来:%v", err)
}
if evidence.ProvenanceSnapshotID != snapshotID || evidence.ProvenanceJobID != first.Backup.WriteSet.JobID {
t.Fatalf("复用证据没有指向前一次成功同步:%+v", evidence)
}
if evidence.TargetRowDigest != evidence.RetargetedRowDigest || evidence.ProvenanceRowDigest != evidence.ArchivedRowDigest {
t.Fatalf("复用证据的行摘要不一致:%+v", evidence)
}
if len(evidence.Objects) != 1 || evidence.Objects[0].SHA256 == "" {
t.Fatalf("复用证据必须逐个记录对象字节:%+v", evidence.Objects)
}
// 复用不许改原行:xmin 与整行 JSON 都必须原样。
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
if afterXmin != beforeXmin {
t.Fatalf("复用资源被重写了:xmin %s -> %s", beforeXmin, afterXmin)
}
if afterRow != beforeRow {
t.Fatalf("复用资源内容变了:\n%s\n->\n%s", beforeRow, afterRow)
}
// 历史孤儿/其它知识库/共享/外租户资源一律不许被覆盖。
for id, want := range map[string]string{"res-orphan": orphanBefore, "res-other": otherBefore,
"res-shared": sharedBefore, "res-foreign": foreignBefore} {
if got := resourceRowOf(t, fx.pool, id); got != want {
t.Fatalf("资源 %s 被改动:\n%s\n->\n%s", id, want, got)
}
}
}
func TestLibraryResyncAfterRestoreRejectsReuseWithoutProvenanceIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, first, _ := syncThenRestore(t, fx, snapshotID)
before := universeDump(t, fx.pool)
// 已登记的来源里没有 res-src(只有这次同步之前的预备份):即使目标那一行
// 看起来一模一样,也不许放行。
noSource := []libraryrelease.PriorSnapshot{{JobID: first.Backup.WriteSet.JobID,
SnapshotID: first.Backup.SnapshotID, Kind: libraryrelease.PriorSnapshotPrebackup}}
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-no-provenance", noSource), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("没有来源证据时必须拒绝:%+v", outcome)
}
if outcome.FailedAt != "" {
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at=%s", outcome.FailedAt)
}
if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "可审计") {
t.Fatalf("拒绝原因必须点名资源并说明缺来源证据:%s", outcome.Error)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
}
}
func TestLibraryResyncAfterRestoreRejectsReuseWhenContentDiffersIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
// 同 ID 同租户,但内容被改过(这里改大小,等价于"另一份文件占了这个 ID")。
if _, err := fx.pool.Exec(fx.ctx, `UPDATE resources SET size=size+1 WHERE id='res-src'`); err != nil {
t.Fatal(err)
}
changed := resourceRowOf(t, fx.pool, "res-src")
before := universeDump(t, fx.pool)
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-content-diff", priors), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("内容不一致必须拒绝:%+v", outcome)
}
if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "内容") {
t.Fatalf("拒绝原因必须点名内容不一致:%s", outcome.Error)
}
if resourceRowOf(t, fx.pool, "res-src") != changed {
t.Fatal("被拒绝的同步不许改目标资源")
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
}
}
// 同租户、同 ID、同内容,但来源快照属于另一个知识库:仍然不许复用。
// "归属无法证明"包括"来源不同",不能因为字节一样就把别人的行当成自己的。
func TestLibraryReuseRequiresSameSourceScopeIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
otherSource := sealSourceSnapshotScope(t, fx, wsSourceTenant+1, "kb-source-other", nil, nil)
before := universeDump(t, fx.pool)
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(otherSource, "job-sync-other-source", priors), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("来源范围不同必须拒绝复用:%+v", outcome)
}
if !strings.Contains(outcome.Error, "可审计") {
t.Fatalf("拒绝原因必须说明来源不可审计:%s", outcome.Error)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
}
}
// 部分写失败:database/resources 已提交、wiki 因重复 slug 失败 → 回滚只清"本次真新建",
// 复用资源与历史孤儿原内容完整保留(xmin 也不变)。
func TestLibraryHalfWriteRollbackKeepsReusedResourcesIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
reuseXmin, reuseRow := resourceFingerprint(t, fx.pool, "res-src")
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
next := sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB,
func(parts map[string][]map[string]any) {
parts["chunks"] = append(parts["chunks"], map[string]any{"id": "chunk-src-2", "tenant_id": wsSourceTenant,
"knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src", "content": "新增正文 resource://" + wsHandleNew})
parts["resources"] = append(parts["resources"], map[string]any{"id": "res-new", "handle": wsHandleNew,
"tenant_id": wsSourceTenant, "storage_backend_id": "backend-source", "provider": "oss",
"physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-new"),
"size": int64(len(resNewBody))})
parts["resource_bindings"] = append(parts["resource_bindings"], map[string]any{"id": "bind-src-2",
"resource_id": "res-new", "tenant_id": wsSourceTenant, "owner_type": "chunk",
"owner_id": "chunk-src-2", "relation": "attachment"})
// 同一作用域两页同 slug:wiki 的唯一索引让 wiki 阶段失败(真实半写现场)。
parts["wiki_pages"] = []map[string]any{
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
}
}, map[string]string{"res-new": resNewBody})
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(next, "job-sync-half-write", priors), target, backups, nil)
if outcome.State != "rolled_back" {
t.Fatalf("wiki 半写必须回滚:%+v", outcome)
}
if outcome.FailedAt != libraryrelease.StageWiki {
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
}
ledger := outcome.Backup.WriteSet
if ledger == nil {
t.Fatal("半写作业必须留下写集合")
}
if created := ledger.CreatedIDs("resources"); len(created) != 1 || created[0] != "res-new" {
t.Fatalf("本次真新建的资源 = %v,want [res-new]", created)
}
if reused := ledger.ReusedIDs("resources"); len(reused) != 1 || reused[0] != "res-src" {
t.Fatalf("复用资源 = %v,want [res-src]", reused)
}
// 本次真新建的清干净。
var residue int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-new'`).Scan(&residue); err != nil {
t.Fatal(err)
}
if residue != 0 {
t.Fatal("回滚后本次新建的资源必须被清理")
}
// 复用资源与历史孤儿原内容(含 xmin)保留。
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
if afterXmin != reuseXmin || afterRow != reuseRow {
t.Fatalf("回滚动了复用资源:xmin %s->%s\n%s\n->\n%s", reuseXmin, afterXmin, reuseRow, afterRow)
}
if resourceRowOf(t, fx.pool, "res-orphan") != orphanBefore {
t.Fatal("回滚动了历史孤儿资源")
}
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
t.Fatalf("回滚必须与预备份一致:%v", err)
}
}