Files
knowledge-governance/internal/releaseadapter/v071/library_reuse_integration_test.go
T
姚章浩 537a6df46d
build / build (push) Successful in 1m39s
fix(library): 首次同步可复用同租户已存在资源
取消资源复用必须属于本目标历史成功同步/预备份的自加门槛。
仍要求目标行逐列一致与实际对象字节一致;旧库资源行/绑定/对象不删不改。
新增 reuse evidence v3:无 provenance,保留行摘要与逐对象字节证据。
2026-11-04 14:22:51 +08:00

389 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"reflect"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryrelease"
)
// 这一组用例覆盖真实缺陷现场:同步 A 成功 → 手工恢复 B(对象保留、资源行不删)
// → 再同步 A。旧预检把"恢复保留下来的、本目标自己上次写进去的资源"判成
// "其它知识库的同 ID 冲突",整个同步在写第一笔之前被拒。
//
// 复用的放行条件必须是可审计的证据,而不是"目标里没人引用":
// - 目标现状那一行与本次要写的行逐列相同(除易变时间戳);
// - 该资源的对象字节与本次快照归档摘要一致;
// - 治理库里存在"这个目标成功同步过、且含有同一行内容"的快照/作业。
// 缺任何一条都必须继续拒绝,且拒绝发生在写入之前。
const (
wsHandleNew = "gggggggggggggggggggggg"
// resNewBody 是"A 的下一次发布"新增资源的内容;与 res-src 的 "12345678" 不同,
// 这样"本次真新建"和"证明可复用"在同一份写集合里能同时被观察到。
resNewBody = "87654321"
)
// reusePriors 是治理库会给出的来源清单:一次成功同步 + 它产生的预备份。
func reusePriors(syncJobID, syncSnapshotID string, prebackups ...string) []libraryrelease.PriorSnapshot {
priors := []libraryrelease.PriorSnapshot{{JobID: syncJobID, SnapshotID: syncSnapshotID, Kind: libraryrelease.PriorSnapshotSync}}
for _, snapshot := range prebackups {
priors = append(priors, libraryrelease.PriorSnapshot{JobID: syncJobID, SnapshotID: snapshot,
Kind: libraryrelease.PriorSnapshotPrebackup})
}
return priors
}
func requestForWithPrior(snapshotID, jobID string, prior []libraryrelease.PriorSnapshot) libraryrelease.Request {
request := requestFor(snapshotID)
request.JobID = jobID
request.PriorSnapshots = prior
return request
}
// syncThenRestore 造出现场:同步 A 成功,然后按生产恢复作业的方式恢复到 A 的
// 预备份(再走一次 Run:新的预备份 + 它自己的写集合)。恢复不删 resources 表,
// 所以 A 写进去的资源会作为"没人引用的行"留在目标里——正是本次缺陷的前提。
func syncThenRestore(t *testing.T, fx *writeSetFixture, snapshotID string) (*ProductionLibraryTarget,
*ProductionBackupStore, libraryrelease.Outcome, []libraryrelease.PriorSnapshot) {
t.Helper()
target := fx.target(t)
backups := fx.backups(t, target)
first := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a", nil), target, backups, nil)
if first.State != "succeeded" {
t.Fatalf("首次同步 A 必须成功:%+v", first)
}
if first.Backup.WriteSet == nil {
t.Fatal("首次同步必须落盘写集合")
}
// 恢复作业同样只认已登记来源:它自己刚拍的那份预备份就是本目标的历史状态。
restore := libraryrelease.Run(fx.ctx, requestForWithPrior(first.Backup.SnapshotID, "job-restore-b",
reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)), target, backups, nil)
if restore.State != "succeeded" {
t.Fatalf("手工恢复 B 必须成功:%+v", restore)
}
var bindings int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resource_bindings WHERE resource_id='res-src'`).Scan(&bindings); err != nil {
t.Fatal(err)
}
if bindings != 0 {
t.Fatalf("恢复后 res-src 不应还有绑定行(现场前提):%d", bindings)
}
return target, backups, first, reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)
}
// resourceFingerprint 取 PG 行的 xmin 与整行 JSON:xmin 不变才能证明"没有重写这一行"。
func resourceFingerprint(t *testing.T, pool *pgxpool.Pool, id string) (string, string) {
t.Helper()
var xmin, row string
if err := pool.QueryRow(t.Context(), `SELECT xmin::text, to_jsonb(r)::text FROM resources r WHERE id=$1`, id).
Scan(&xmin, &row); err != nil {
t.Fatalf("读取资源 %s: %v", id, err)
}
return xmin, row
}
func resourceRowOf(t *testing.T, pool *pgxpool.Pool, id string) string {
t.Helper()
var row string
if err := pool.QueryRow(t.Context(), `SELECT to_jsonb(r)::text FROM resources r WHERE id=$1`, id).Scan(&row); err != nil {
t.Fatalf("读取资源 %s: %v", id, err)
}
return row
}
func TestLibraryResyncAfterRestoreReusesProvenResourcesIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, first, priors := syncThenRestore(t, fx, snapshotID)
beforeXmin, beforeRow := resourceFingerprint(t, fx.pool, "res-src")
// B10:no-op 复用不允许重写对象——精确版本与字节都必须原样。
reusedObjectPath := wsLocator("backend-target", "minio", wsBucket, wsTargetTenant, "res-src")
objectBeforeReuse, err := fx.objects.s3.ObserveObject(fx.ctx, "minio", reusedObjectPath)
if err != nil || !objectBeforeReuse.Exists {
t.Fatalf("复用对象不存在:%+v %v", objectBeforeReuse, err)
}
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
otherBefore := resourceRowOf(t, fx.pool, "res-other")
sharedBefore := resourceRowOf(t, fx.pool, "res-shared")
foreignBefore := resourceRowOf(t, fx.pool, "res-foreign")
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a-again", priors),
target, backups, nil)
if outcome.State != "succeeded" {
t.Fatalf("恢复后再次同步 A 必须完成:%+v", outcome)
}
if outcome.Backup.WriteSet == nil {
t.Fatal("写集合必须落盘")
}
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
if err != nil {
t.Fatalf("写集合记录取不回来:%v", err)
}
reused := ledger.ReusedIDs("resources")
if len(reused) != 1 || reused[0] != "res-src" {
t.Fatalf("复用资源 = %v,want [res-src]", reused)
}
if created := ledger.CreatedIDs("resources"); len(created) != 0 {
t.Fatalf("复用资源不能被记成本次新建(恢复会删掉它):%v", created)
}
var reuseRow libraryrelease.WriteSetRow
for _, row := range ledger.Rows {
if row.Table == "resources" && row.ID == "res-src" {
reuseRow = row
}
}
if !reuseRow.Existing || !reuseRow.Reused || reuseRow.ReuseEvidence == "" {
t.Fatalf("复用行必须带证据摘要并标记为写前已存在:%+v", reuseRow)
}
evidence, err := LoadReuseEvidence(fx.archiveRoot, reuseRow.ReuseEvidence)
if err != nil {
t.Fatalf("复用证据取不回来:%v", err)
}
if evidence.ProvenanceSnapshotID != snapshotID || evidence.ProvenanceJobID != first.Backup.WriteSet.JobID {
t.Fatalf("复用证据没有指向前一次成功同步:%+v", evidence)
}
if evidence.TargetRowDigest != evidence.RetargetedRowDigest || evidence.ProvenanceRowDigest != evidence.ArchivedRowDigest {
t.Fatalf("复用证据的行摘要不一致:%+v", evidence)
}
if len(evidence.Objects) != 1 || evidence.Objects[0].SHA256 == "" {
t.Fatalf("复用证据必须逐个记录对象字节:%+v", evidence.Objects)
}
// 复用不许改原行:xmin 与整行 JSON 都必须原样。
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
if afterXmin != beforeXmin {
t.Fatalf("复用资源被重写了:xmin %s -> %s", beforeXmin, afterXmin)
}
if afterRow != beforeRow {
t.Fatalf("复用资源内容变了:\n%s\n->\n%s", beforeRow, afterRow)
}
objectAfterReuse, err := fx.objects.s3.ObserveObject(fx.ctx, "minio", reusedObjectPath)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(objectBeforeReuse, objectAfterReuse) {
t.Fatalf("no-op 复用改写了对象版本:%+v -> %+v", objectBeforeReuse, objectAfterReuse)
}
// 历史孤儿/其它知识库/共享/外租户资源一律不许被覆盖。
for id, want := range map[string]string{"res-orphan": orphanBefore, "res-other": otherBefore,
"res-shared": sharedBefore, "res-foreign": foreignBefore} {
if got := resourceRowOf(t, fx.pool, id); got != want {
t.Fatalf("资源 %s 被改动:\n%s\n->\n%s", id, want, got)
}
}
}
func TestLibraryResyncAfterRestoreReusesTargetRowWithoutProvenanceIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, first, _ := syncThenRestore(t, fx, snapshotID)
beforeXmin, beforeRow := resourceFingerprint(t, fx.pool, "res-src")
// 用户批准:没有本目标历史来源也可以复用,条件退回到“目标行与本次映射行逐列一致
// + 对象字节与归档一致”。这里的 res-src 是上一次同步写进目标、恢复后留下的行。
noSource := []libraryrelease.PriorSnapshot{{JobID: first.Backup.WriteSet.JobID,
SnapshotID: first.Backup.SnapshotID, Kind: libraryrelease.PriorSnapshotPrebackup}}
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-no-provenance", noSource), target, backups, nil)
if outcome.State != "succeeded" {
t.Fatalf("无本目标历史来源、但行与字节一致时必须复用成功:%+v", outcome)
}
if xmin, row := resourceFingerprint(t, fx.pool, "res-src"); xmin != beforeXmin || row != beforeRow {
t.Fatalf("复用不得改写共享资源行:\n%s\n->\n%s", beforeRow, row)
}
}
func TestLibraryResyncAfterRestoreRejectsReuseWhenContentDiffersIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
// 同 ID 同租户,但内容被改过(这里改大小,等价于"另一份文件占了这个 ID")。
if _, err := fx.pool.Exec(fx.ctx, `UPDATE resources SET size=size+1 WHERE id='res-src'`); err != nil {
t.Fatal(err)
}
changed := resourceRowOf(t, fx.pool, "res-src")
before := universeDump(t, fx.pool)
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-content-diff", priors), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("内容不一致必须拒绝:%+v", outcome)
}
if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "内容") {
t.Fatalf("拒绝原因必须点名内容不一致:%s", outcome.Error)
}
if resourceRowOf(t, fx.pool, "res-src") != changed {
t.Fatal("被拒绝的同步不许改目标资源")
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
}
}
// 同租户、同 ID、同内容,来源快照属于另一个知识库:历史归属不再参与放行,
// 但仍必须以“目标行逐列一致 + 对象字节一致”为前提,且不得改写旧库行。
// 若内容或字节不同,下面的负例必须继续拒绝。
func TestLibraryReuseAcrossSourceScopeKeepsTargetRowsIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
otherSource := sealSourceSnapshotScope(t, fx, wsSourceTenant+1, "kb-source-other", nil, nil)
before := universeDump(t, fx.pool)
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(otherSource, "job-sync-other-source", priors), target, backups, nil)
if outcome.State != "succeeded" {
t.Fatalf("同内容同字节的跨来源复用应被允许:%+v", outcome)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff == "" {
// 允许成功但必须有实际导入结果;目标已有行可能被保留,不能空跑成功。
t.Fatal("复用后目标应出现新的导入结果")
}
}
// 显式注入晚期 Wiki 故障:database/resources 已提交,回滚只清"本次真新建",
// 复用资源与历史孤儿原内容完整保留(xmin 也不变)。
func TestLibraryHalfWriteRollbackKeepsReusedResourcesIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
reuseXmin, reuseRow := resourceFingerprint(t, fx.pool, "res-src")
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
next := sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB,
func(parts map[string][]map[string]any) {
parts["chunks"] = append(parts["chunks"], map[string]any{"id": "chunk-src-2", "tenant_id": wsSourceTenant,
"knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src", "content": "新增正文 resource://" + wsHandleNew})
parts["resources"] = append(parts["resources"], map[string]any{"id": "res-new", "handle": wsHandleNew,
"tenant_id": wsSourceTenant, "storage_backend_id": "backend-source", "provider": "oss",
"physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-new"),
"size": int64(len(resNewBody))})
parts["resource_bindings"] = append(parts["resource_bindings"], map[string]any{"id": "bind-src-2",
"resource_id": "res-new", "tenant_id": wsSourceTenant, "owner_type": "chunk",
"owner_id": "chunk-src-2", "relation": "attachment"})
}, map[string]string{"res-new": resNewBody})
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(next, "job-sync-half-write", priors), contractFailWikiTarget{target}, backups, nil)
if outcome.State != "rolled_back" {
t.Fatalf("wiki 半写必须回滚:%+v", outcome)
}
if outcome.FailedAt != libraryrelease.StageWiki {
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
}
ledger := outcome.Backup.WriteSet
if ledger == nil {
t.Fatal("半写作业必须留下写集合")
}
if created := ledger.CreatedIDs("resources"); len(created) != 1 || created[0] != "res-new" {
t.Fatalf("本次真新建的资源 = %v,want [res-new]", created)
}
if reused := ledger.ReusedIDs("resources"); len(reused) != 1 || reused[0] != "res-src" {
t.Fatalf("复用资源 = %v,want [res-src]", reused)
}
// 本次真新建的清干净。
var residue int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-new'`).Scan(&residue); err != nil {
t.Fatal(err)
}
if residue != 0 {
t.Fatal("回滚后本次新建的资源必须被清理")
}
// 复用资源与历史孤儿原内容(含 xmin)保留。
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
if afterXmin != reuseXmin || afterRow != reuseRow {
t.Fatalf("回滚动了复用资源:xmin %s->%s\n%s\n->\n%s", reuseXmin, afterXmin, reuseRow, afterRow)
}
if resourceRowOf(t, fx.pool, "res-orphan") != orphanBefore {
t.Fatal("回滚动了历史孤儿资源")
}
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
t.Fatalf("回滚必须与预备份一致:%v", err)
}
}
// 新目标库首次同步:本目标没有任何历史成功同步/预备份来源,但同 tenant 的
// 另一个知识库已经绑定了一个 ID、整行内容与对象字节都完全相同的资源。
// 用户已批准取消"必须有本目标历史来源"这一自加门槛;仍须保留旧库行/绑定/对象,
// 且内容不同或字节不同必须继续拒绝。
func TestLibraryFirstSyncReusesResourceOwnedByOtherKBIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
// 同 tenant 另一个 KB 的文档 + 资源 + 绑定;res-src 与源快照同 ID 同内容。
otherDoc := "doc-first-other"
statements := []string{
`INSERT INTO knowledges VALUES('` + otherDoc + `',10001,'` + wsOtherKB + `','completed',0,'',NULL)`,
`INSERT INTO resources VALUES('res-src','` + wsHandleSource + `',10001,'` + wsLocator("backend-target", "minio", wsBucket, 10001, "res-src") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-first-other',10001,'res-src','knowledge','` + otherDoc + `',NULL)`,
}
for _, statement := range statements {
if _, err := fx.pool.Exec(fx.ctx, statement); err != nil {
t.Fatalf("seed other-KB resource: %v\n%s", err, statement)
}
}
if err := fx.objects.s3.Put(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, 10001, "res-src"),
strings.NewReader("12345678"), 8); err != nil {
t.Fatal(err)
}
beforeXmin, beforeRow := resourceFingerprint(t, fx.pool, "res-src")
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-first-no-provenance", nil), target, backups, nil)
if outcome.State != "succeeded" {
t.Fatalf("首次同步必须完成(无本目标历史来源,但同 tenant 同内容资源可复用):%+v", outcome)
}
if xmin, row := resourceFingerprint(t, fx.pool, "res-src"); xmin != beforeXmin || row != beforeRow {
t.Fatalf("复用不得改写共享资源行:\n%s\n->\n%s", beforeRow, row)
}
var binding int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resource_bindings WHERE id='bind-first-other' AND resource_id='res-src'`).Scan(&binding); err != nil || binding != 1 {
t.Fatalf("旧库绑定必须保留:%d %v", binding, err)
}
var targetDocs int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM knowledges WHERE knowledge_base_id=$1 AND id<>$2`,
wsTargetKB, otherDoc).Scan(&targetDocs); err != nil || targetDocs == 0 {
t.Fatalf("目标 KB 必须完成导入:%d %v", targetDocs, err)
}
}
// 首次同步复用仍必须核对实际对象字节:行投影一致、但目标对象字节不同 -> 拒绝,
// 且不得改写资源行/绑定或留下写入。
func TestLibraryFirstSyncRejectsReuseWhenObjectBytesDifferIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
otherDoc := "doc-byte-other"
statements := []string{
`INSERT INTO knowledges VALUES('` + otherDoc + `',10001,'` + wsOtherKB + `','completed',0,'',NULL)`,
`INSERT INTO resources VALUES('res-src','` + wsHandleSource + `',10001,'` + wsLocator("backend-target", "minio", wsBucket, 10001, "res-src") + `',NULL,'minio',8,'backend-target')`,
`INSERT INTO resource_bindings VALUES('bind-byte-other',10001,'res-src','knowledge','` + otherDoc + `',NULL)`,
}
for _, statement := range statements {
if _, err := fx.pool.Exec(fx.ctx, statement); err != nil {
t.Fatalf("seed: %v", err)
}
}
if err := fx.objects.s3.Put(fx.ctx, "minio", wsLocator("backend-target", "minio", wsBucket, 10001, "res-src"),
strings.NewReader("87654321"), 8); err != nil {
t.Fatal(err)
}
beforeXmin, beforeRow := resourceFingerprint(t, fx.pool, "res-src")
before := universeDump(t, fx.pool)
snapshotID := sealSourceSnapshot(t, fx, nil)
target := fx.target(t)
backups := fx.backups(t, target)
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-byte-diff", nil), target, backups, nil)
if outcome.State != "failed" || !strings.Contains(outcome.Error, "字节") {
t.Fatalf("对象字节不同必须拒绝:%+v", outcome)
}
if xmin, row := resourceFingerprint(t, fx.pool, "res-src"); xmin != beforeXmin || row != beforeRow {
t.Fatal("拒绝时不得改写共享资源行")
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("拒绝时不得写入目标:%s", diff)
}
}