fix(library): 恢复保留下来的资源凭可审计证据复用

同步 A 成功后手工恢复 B,resources 表既不删行也不删对象;再同步 A 时,
resources 里那 21 行以“同租户但不在预备份范围”被判成别人的知识库,整次
同步在写第一笔之前被拒。

“目标里没人引用”不是归属证据:历史孤儿、其它知识库暂未绑定的资源都满足
它。改成唯一放行路径——可审计复用(v071/library_reuse.go):
1. 持久来源:这个目标成功同步过且登记过生效版本的快照,或它自己的预备份,
   且同一资源行在来源里内容一致;
2. 整行内容(除易变时间戳)与本次写入将要落地的行完全一致;
3. 该资源的对象字节与本次快照归档摘要一致。

三条同时成立才允许复用:写入阶段跳过它(不重写行、不重传对象),写集合里
记 existing+reused,恢复不清理;证据单独落盘 reuse/<digest>.json,摘要写进
写集合行,不冒充历史台账。内容不同、来源范围不同、或没有持久来源一律继续
拒绝,且拒绝发生在写入之前。

为什么把预备份也算来源:恢复作业的写集合同样会把“本目标历史状态里已有的
资源”判成跨范围冲突,而它们本来就出自这个目标自己的快照。

验证:隔离 PG+ES+MinIO 真服务下新增 5 个场景用例(复用成功且 xmin 不变、
无来源拒绝、内容不同拒绝、同内容不同来源拒绝、wiki 半写回滚只清本次新建);
v071 整包与全仓 go test 通过;make verify 15 项全绿。
This commit is contained in:
姚章浩
2026-10-30 01:59:32 +08:00
parent cf3f4662cc
commit 0252492e53
12 changed files with 1034 additions and 18 deletions
+19 -1
View File
@@ -69,7 +69,13 @@ func (t *contentTrackedTarget) PlanWriteSet(ctx context.Context, plan libraryrel
if !ok {
return libraryrelease.WriteSet{}, errors.New("目标实现不支持写集合预检:拒绝在没有可追溯写集合的情况下写生产")
}
return planner.PlanWriteSet(ctx, plan)
set, err := planner.PlanWriteSet(ctx, plan)
if err == nil {
// 日志是"wrapper 路径真的执行了预检"的现场证据:接管排查先看这一行。
log.Printf("library writeset planned job=%s target=%s digest=%s rows=%d reused=%d target_snapshot=%s",
set.JobID, set.Target, set.Digest, len(set.Rows), len(set.ReusedIDs("resources")), set.TargetSnapshotID)
}
return set, err
}
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
@@ -157,6 +163,17 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
if err != nil || !found {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "没有与目标快照匹配的有效发布授权", "prebackup")
}
// 资源复用的来源证据:只认这个目标"成功同步过且登记过生效版本"的快照与它自己
// 的预备份。读不到就按空处理——那只会让复用被拒绝,绝不会误放行覆盖。
priorSnapshots := make([]libraryrelease.PriorSnapshot, 0, 8)
if sources, sourceErr := store.LibraryReuseSources(ctx, payload.Scope, 100); sourceErr != nil {
log.Printf("library %s prior reuse sources unavailable: %v", payload.LibraryJobID, sourceErr)
} else {
for _, source := range sources {
priorSnapshots = append(priorSnapshots, libraryrelease.PriorSnapshot{JobID: source.JobID,
SnapshotID: source.SnapshotID, Kind: source.Kind, FinishedAt: source.FinishedAt})
}
}
// 生产索引映射基线:优先当前生效版本,其次任何一次成功预备份(它也记录了生产实际映射)。
// 只认"生效版本"会在恢复把版本 supersede 掉之后误判"索引被改动"。
knownMapping := ""
@@ -325,6 +342,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope,
SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now,
PriorSnapshots: priorSnapshots,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
+1
View File
@@ -84,6 +84,7 @@ func writeSetEvidence(existing map[string]any, set libraryrelease.WriteSet, mapp
evidence["writeset_universe_digest"] = set.UniverseDigest()
evidence["writeset_target_snapshot"] = set.TargetSnapshotID
evidence["writeset_rows"] = len(set.Rows)
evidence["writeset_reused_rows"] = len(set.ReusedIDs("resources"))
return evidence
}
+9 -2
View File
@@ -167,10 +167,12 @@ func (t *plainTarget) Restore(context.Context, libraryrelease.BackupRef) error
type planningTarget struct {
plainTarget
planned int
last libraryrelease.WriteSetPlan
}
func (t *planningTarget) PlanWriteSet(context.Context, libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
func (t *planningTarget) PlanWriteSet(_ context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
t.planned++
t.last = plan
return libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: "job-1", Target: "internal"}, nil
}
@@ -181,12 +183,17 @@ func TestContentTrackedTargetForwardsWriteSetPlanning(t *testing.T) {
if !ok {
t.Fatal("包装后的目标必须仍然实现 WriteSetPlanner,否则写集合会被静默跳过")
}
if _, err := planner.PlanWriteSet(context.Background(), libraryrelease.WriteSetPlan{JobID: "job-1", Target: "internal"}); err != nil {
if _, err := planner.PlanWriteSet(context.Background(), libraryrelease.WriteSetPlan{JobID: "job-1", Target: "internal",
PriorSnapshots: []libraryrelease.PriorSnapshot{{JobID: "job-0", SnapshotID: "snap-0", Kind: libraryrelease.PriorSnapshotSync}}}); err != nil {
t.Fatalf("plan: %v", err)
}
if inner.planned != 1 {
t.Fatalf("预检必须落到内层目标,planned=%d", inner.planned)
}
// 来源证据也必须透传:丢掉它就会让"恢复保留下来的资源"永远无法证明可复用。
if len(inner.last.PriorSnapshots) != 1 || inner.last.PriorSnapshots[0].SnapshotID != "snap-0" {
t.Fatalf("PriorSyncs 没有透传到内层:%+v", inner.last)
}
// 内层不支持预检 ⇒ 硬错误,不许"没有台账也照写"。
strict := &contentTrackedTarget{inner: &plainTarget{}, onApply: func() {}}
+6 -1
View File
@@ -45,6 +45,10 @@ type Request struct {
ParamsDigest string
Grant Grant
Now time.Time
// PriorSnapshots 是这个目标的持久来源(成功同步过的快照 + 自己的预备份),
// 由治理库提供。资源复用只认它作证据;为空时"目标里已有但不在预备份范围"
// 的资源一律按冲突拒绝。
PriorSnapshots []PriorSnapshot
}
// CurrentState 是生产当前整库的指纹,用于预备份与最终验证。
@@ -213,7 +217,8 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor
if backup.WriteSet == nil {
if planner, ok := target.(WriteSetPlanner); ok {
set, planErr := planner.PlanWriteSet(ctx, WriteSetPlan{JobID: request.JobID, Target: request.Target,
BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: request.SnapshotID, Universe: backup.Universe})
BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: request.SnapshotID, Universe: backup.Universe,
PriorSnapshots: request.PriorSnapshots})
if planErr != nil {
outcome.State = "failed"
outcome.Error = "写集合/冲突预检失败,未对生产执行任何写入:" + planErr.Error()
+54
View File
@@ -28,6 +28,16 @@ type WriteSetRow struct {
Existing bool `json:"existing"`
// Owner 是写前该行的归属("<tenant>/<knowledge_base>"),不存在时为空。
Owner string `json:"owner,omitempty"`
// Reused 表示这一行虽然写前就在目标里、却不在预备份范围内,但已经用
// "持久作业/快照证据 + 整行内容 + 对象字节"证明与本次要写的内容完全一致,
// 因此本次既不重写它、也不在恢复时清理它。
//
// 不能只因为"它没被任何绑定/正文引用"就当成自己的:历史孤儿、同租户另一个
// 知识库暂未绑定的资源同样满足"没人引用",覆盖或删除同样是数据丢失。
Reused bool `json:"reused,omitempty"`
// ReuseEvidence 是可复用证据文档的摘要(reuse/<digest>.json),独立于写集合,
// 防止把这次推断伪装成历史台账。
ReuseEvidence string `json:"reuse_evidence,omitempty"`
}
// WriteSetPlan 是调用方在写第一笔之前交给目标的信息:绑定作业、目标、预备份与快照,
@@ -38,6 +48,29 @@ type WriteSetPlan struct {
BackupSnapshotID string
TargetSnapshotID string
Universe map[string]TableFingerprint
// PriorSnapshots 是治理库里"这个目标的持久来源":成功同步过的快照,以及这个
// 目标自己的预备份。资源复用必须沿它把来源追到真实登记过的快照,而不是按
// "目标里没人引用"推断归属。
PriorSnapshots []PriorSnapshot
}
// 复用来源的两种合法类别。
const (
// PriorSnapshotSync:这个目标成功同步过这份快照(且登记了生效版本)。
PriorSnapshotSync = "sync"
// PriorSnapshotPrebackup:这份快照是从这个目标自己身上拍的预备份,把它写回去
// 就是回到本目标的历史状态。
PriorSnapshotPrebackup = "prebackup"
)
// PriorSnapshot 是一次**已登记**的来源:作业身份 + 快照身份 + 来源类别。
// 只有它也含有同一个资源 ID、且行内容完全一致,才允许把目标里那行当作
// "本次写入的等价物"。
type PriorSnapshot struct {
JobID string `json:"job_id"`
SnapshotID string `json:"snapshot_id"`
Kind string `json:"kind"`
FinishedAt time.Time `json:"finished_at,omitempty"`
}
// WriteSet 是"这一次发布的写集合 + 写前整实例内容证据",内容寻址、可校验、
@@ -124,6 +157,14 @@ func (w *WriteSet) Verify() error {
if w.Digest == "" {
return errors.New("写集合记录没有摘要")
}
for _, row := range w.Rows {
if row.Reused && !row.Existing {
return fmt.Errorf("写集合记录把 %s/%s 标成可复用却不是写前已存在行", row.Table, row.ID)
}
if row.Reused && row.ReuseEvidence == "" {
return fmt.Errorf("写集合记录把 %s/%s 标成可复用却没有证据摘要", row.Table, row.ID)
}
}
digest, err := w.computeDigest()
if err != nil {
return err
@@ -158,6 +199,19 @@ func (w WriteSet) ExistingIDs(table string) []string {
return ids
}
// ReusedIDs 返回本次"证明可复用、因此既不重写也不清理"的精确身份。
// 它与 CreatedIDs 互斥,恢复清理一律不看这里(复用行必须原样保留)。
func (w WriteSet) ReusedIDs(table string) []string {
ids := make([]string, 0, 8)
for _, row := range w.Rows {
if row.Table == table && row.Reused {
ids = append(ids, row.ID)
}
}
sort.Strings(ids)
return ids
}
// UniverseDigest 是整实例证据的摘要,随台账一起登记,便于独立核对。
func (w WriteSet) UniverseDigest() string {
if len(w.Universe) == 0 {
+62
View File
@@ -102,3 +102,65 @@ func TestWriteSetCreatedIDsOnlyCoversRowsCreatedByThisRelease(t *testing.T) {
t.Fatal("universe digest must be derived from the whole-instance evidence")
}
}
// 复用行是"写前已存在 + 有证据摘要"的一种:既不算本次新建(恢复不许删),
// 也不能没有证据摘要,更不能被标成新建。
func TestWriteSetReusedRowsStayOutOfCleanupAndRequireEvidence(t *testing.T) {
set := WriteSet{Version: WriteSetVersion, JobID: "job-2", Target: "internal", TenantID: 10001,
KnowledgeBaseID: "kb", BackupSnapshotID: "backup-2", TargetSnapshotID: "snap-2",
CreatedAt: time.Date(2026, 10, 30, 1, 0, 0, 0, time.UTC),
Rows: []WriteSetRow{
{Table: "resources", ID: "res-reused", Existing: true, Owner: "10001/", Reused: true, ReuseEvidence: "digest-1"},
{Table: "resources", ID: "res-new"},
},
Universe: map[string]TableFingerprint{"resources": {Count: 3, Digest: "abc"}}}
if err := set.Seal(); err != nil {
t.Fatal(err)
}
if err := set.Verify(); err != nil {
t.Fatalf("复用行必须能通过校验: %v", err)
}
if created := set.CreatedIDs("resources"); len(created) != 1 || created[0] != "res-new" {
t.Fatalf("复用行被当成本次新建(恢复会删掉它):%v", created)
}
if reused := set.ReusedIDs("resources"); len(reused) != 1 || reused[0] != "res-reused" {
t.Fatalf("reused resources = %v", reused)
}
noEvidence := set
noEvidence.Rows = append([]WriteSetRow{}, set.Rows...)
for index := range noEvidence.Rows {
if noEvidence.Rows[index].ID == "res-reused" {
noEvidence.Rows[index].ReuseEvidence = ""
}
}
if err := noEvidence.Seal(); err != nil {
t.Fatal(err)
}
if err := noEvidence.Verify(); err == nil {
t.Fatal("复用行没有证据摘要时必须校验失败")
}
notExisting := set
notExisting.Rows = append([]WriteSetRow{}, set.Rows...)
for index := range notExisting.Rows {
if notExisting.Rows[index].ID == "res-reused" {
notExisting.Rows[index].Existing = false
}
}
if err := notExisting.Seal(); err != nil {
t.Fatal(err)
}
if err := notExisting.Verify(); err == nil {
t.Fatal("复用行必须同时是写前已存在行(否则会被恢复当残渣)")
}
// 旧记录(没有复用字段)仍然必须能读回:格式演进不能把历史台账判成损坏。
legacy := sampleWriteSet()
if err := legacy.Seal(); err != nil {
t.Fatal(err)
}
if err := legacy.Verify(); err != nil {
t.Fatalf("历史写集合记录必须继续有效: %v", err)
}
}
@@ -102,6 +102,50 @@ FROM library_jobs WHERE id=$1`, id).Scan(&job.ID, &job.Scope, &kind, &state, &jo
return job, nil
}
// LibraryReuseSource 是一条"这个目标的持久来源"记录:成功同步过的快照,
// 或这个目标自己的预备份。资源复用只认它作来源证据。
type LibraryReuseSource struct {
JobID string
SnapshotID string
Kind string // sync | prebackup
FinishedAt time.Time
}
// LibraryReuseSources 读这个目标的持久来源(含已被恢复顶替的生效版本)。
// 同步来源要求同时存在 production_effective 版本登记:只写成功但版本没登记的
// 作业不算可审计来源。
func (s *Store) LibraryReuseSources(ctx context.Context, scope string, limit int) ([]LibraryReuseSource, error) {
if limit < 1 || limit > 200 {
limit = 100
}
rows, err := s.db.Query(ctx, `
SELECT j.id, j.target_snapshot_id, 'sync', coalesce(j.finished_at, j.updated_at)
FROM library_jobs j
WHERE j.scope=$1 AND j.kind='sync' AND j.state='succeeded' AND coalesce(j.target_snapshot_id,'')<>''
AND EXISTS (SELECT 1 FROM library_versions v
WHERE v.scope=j.scope AND v.kind='production_effective'
AND v.snapshot_id=j.target_snapshot_id AND v.source_job_id=j.id)
UNION ALL
SELECT coalesce(v.source_job_id,''), v.snapshot_id, 'prebackup', v.created_at
FROM library_versions v
WHERE v.scope=$1 AND v.kind='production_prebackup' AND coalesce(v.snapshot_id,'')<>''
ORDER BY 4 DESC
LIMIT $2`, scope, limit)
if err != nil {
return nil, fmt.Errorf("load library reuse sources: %w", err)
}
defer rows.Close()
sources := make([]LibraryReuseSource, 0, 8)
for rows.Next() {
var source LibraryReuseSource
if err := rows.Scan(&source.JobID, &source.SnapshotID, &source.Kind, &source.FinishedAt); err != nil {
return nil, fmt.Errorf("scan library reuse source: %w", err)
}
sources = append(sources, source)
}
return sources, rows.Err()
}
func (s *Store) libraryStages(ctx context.Context, id string) ([]governance.LibraryStage, error) {
rows, err := s.db.Query(ctx, `
SELECT stage, state, total, completed, bytes_total, bytes_done, detail, updated_at
@@ -94,6 +94,9 @@ type ProductionLibraryTarget struct {
snapshotKB string
// targetEmbeddingModel 是目标知识库绑定的 embedding 模型 id(文档行要换成它)。
targetEmbeddingModel string
// reusedResources 是"写前已存在、且已证明可复用"的资源身份:写入阶段跳过它们,
// 既不重写行也不重传对象。PlanWriteSet 与 Apply 在同一个作业里串行执行。
reusedResources map[string]bool
}
// resolveStorageBackend 在目标库里找本次写入的存储后端。找不到就报错:宁可不写,
@@ -383,6 +386,14 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc
if len(row) == 0 {
continue
}
// 已证明与目标现状完全一致的资源:不重写这一行(连 updated_at 都不碰)。
// 它在写集合里是 Existing+Reused,恢复清理也不会看它。
if table == resourceReuseTableName && t.reusedResources != nil {
var identity string
if json.Unmarshal(row["id"], &identity) == nil && t.reusedResources[identity] {
continue
}
}
// 行必须先映射到目标范围再落库:原样插入会把行写进来源租户/知识库,
// 目标范围的删除、校验与恢复都看不见它们。
if err := t.retargetRow(table, row); err != nil {
@@ -465,6 +476,11 @@ func (t *ProductionLibraryTarget) importResources(ctx context.Context, archive *
if !ok || location.Path == "" {
return fmt.Errorf("生产缺少资源 %s 的存储路径", part.Name)
}
// 复用资源的对象字节已经在计划阶段逐字节比过:不再重传(重传只会改
// 对象的写入时间,没有任何收益)。
if t.reusedResources[part.Name] {
continue
}
reader, err := archive.OpenPart(part)
if err != nil {
return err
@@ -0,0 +1,444 @@
package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
)
// 资源复用证据:目标里已经有这个资源 ID、但它不在预备份范围内时,唯一安全的
// 处理是证明"本次要写的那一行/那一份对象字节,已经原样存在于目标里"。
//
// 为什么不能按"没人引用"推断归属:历史孤儿、同租户另一个知识库暂未绑定的资源
// 同样没人引用;按它放行等于允许覆盖别人的数据。这里的证据必须同时满足三件事:
// 1. 有可审计的持久来源(治理库里这个目标成功同步过的快照或它自己的预备份,
// 且同一资源行内容一致);
// 2. 整行内容(除易变时间戳外)与本次写入将要落地的行完全一致;
// 3. 该资源的对象字节与本次快照归档摘要一致。
//
// 证据独立落盘(reuse/<digest>.json),摘要写进写集合行;它不冒充历史写集合——
// 那两次没有 writeset 的成功作业依然没有 writeset。
const (
reuseDirName = "reuse"
reuseEvidenceVersion = 1
resourceReuseTableName = "resources"
)
// reuseObject 是一个资源对象的字节证据:归档摘要 + 目标现状必须匹配。
type reuseObject struct {
Name string `json:"name"`
Path string `json:"path"`
Bytes int64 `json:"bytes"`
SHA256 string `json:"sha256"`
}
// ResourceReuseEvidence 是"这一行可以复用"的完整证据文档。
// Digest 覆盖除自己以外的全部内容:被改动就对不上写集合行里的摘要。
type ResourceReuseEvidence struct {
Version int `json:"version"`
JobID string `json:"job_id"`
Target string `json:"target"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
SourceTenantID uint64 `json:"source_tenant_id"`
SourceKnowledgeBaseID string `json:"source_knowledge_base_id"`
TargetSnapshotID string `json:"target_snapshot_id"`
ResourceID string `json:"resource_id"`
// ArchivedRowDigest 是来源快照里那一行(不含易变时间戳)的摘要。
ArchivedRowDigest string `json:"archived_row_digest"`
// RetargetedRowDigest 是同一行按本次目标范围/存储改写后的摘要。
RetargetedRowDigest string `json:"retargeted_row_digest"`
// TargetRowDigest 是目标现状按归档列投影后的摘要:必须与上一条一致。
TargetRowDigest string `json:"target_row_digest"`
// ProvenanceJobID / ProvenanceSnapshotID 指向一次**已成功**的同步作业及
// 它当时写入目标的快照;ProvenanceRowDigest 是那份快照里同一行的摘要。
ProvenanceJobID string `json:"provenance_job_id"`
ProvenanceSnapshotID string `json:"provenance_snapshot_id"`
ProvenanceKind string `json:"provenance_kind"`
ProvenanceRowDigest string `json:"provenance_row_digest"`
Objects []reuseObject `json:"objects,omitempty"`
CreatedAt time.Time `json:"created_at"`
Digest string `json:"digest"`
}
func reusePath(archiveRoot, digest string) string {
return filepath.Join(archiveRoot, reuseDirName, digest+".json")
}
// Seal 规范化并写入摘要;字段顺序由 json.Marshal 的结构体顺序固定。
func (e *ResourceReuseEvidence) Seal() error {
copied := *e
copied.Digest = ""
raw, err := json.Marshal(copied)
if err != nil {
return fmt.Errorf("编码复用证据失败: %w", err)
}
sum := sha256.Sum256(raw)
e.Digest = hex.EncodeToString(sum[:])
return nil
}
// Verify 校验版本、必备身份与摘要;缺任何一项都不许拿它去放行覆盖。
func (e *ResourceReuseEvidence) Verify() error {
if e.Version != reuseEvidenceVersion {
return fmt.Errorf("复用证据版本 %d 不受支持(当前 %d)", e.Version, reuseEvidenceVersion)
}
if e.JobID == "" || e.Target == "" || e.TargetSnapshotID == "" || e.ResourceID == "" ||
e.ProvenanceJobID == "" || e.ProvenanceSnapshotID == "" || e.ProvenanceKind == "" {
return errors.New("复用证据缺少作业/目标/快照/来源身份")
}
if e.TenantID == 0 || e.KnowledgeBaseID == "" || e.SourceTenantID == 0 || e.SourceKnowledgeBaseID == "" {
return errors.New("复用证据缺少范围身份")
}
// 三份行摘要必须一致:来源行、映射后的行、目标现状是同一份内容。
if e.RetargetedRowDigest == "" || e.TargetRowDigest != e.RetargetedRowDigest {
return errors.New("复用证据里目标行与本次要写的行不一致")
}
if e.ProvenanceRowDigest != e.ArchivedRowDigest {
return errors.New("复用证据里的来源行与本次快照行不一致")
}
copied := *e
copied.Digest = ""
raw, err := json.Marshal(copied)
if err != nil {
return err
}
sum := sha256.Sum256(raw)
if hex.EncodeToString(sum[:]) != e.Digest {
return errors.New("复用证据摘要不符(内容被改动或截断)")
}
return nil
}
// PersistReuseEvidence 按内容寻址落盘(只读、幂等)。同摘要已存在时校验后直接返回。
func PersistReuseEvidence(archiveRoot string, evidence *ResourceReuseEvidence) (string, error) {
if evidence.Digest == "" {
if err := evidence.Seal(); err != nil {
return "", err
}
}
if evidence.Digest == "" {
return "", errors.New("复用证据没有摘要,拒绝落盘")
}
directory := filepath.Join(archiveRoot, reuseDirName)
if err := os.MkdirAll(directory, 0700); err != nil {
return "", fmt.Errorf("创建复用证据目录失败: %w", err)
}
path := reusePath(archiveRoot, evidence.Digest)
if existing, readErr := os.ReadFile(path); readErr == nil {
var recorded ResourceReuseEvidence
if json.Unmarshal(existing, &recorded) != nil || recorded.Verify() != nil || recorded.Digest != evidence.Digest {
return "", errors.New("复用证据同名但内容不符,拒绝覆盖")
}
return path, nil
} else if !errors.Is(readErr, os.ErrNotExist) {
return "", fmt.Errorf("读取已有复用证据失败: %w", readErr)
}
raw, err := json.Marshal(*evidence)
if err != nil {
return "", fmt.Errorf("编码复用证据失败: %w", err)
}
temporary := path + ".tmp"
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return "", fmt.Errorf("写入复用证据失败: %w", err)
}
if _, err := file.Write(raw); err != nil {
file.Close()
os.Remove(temporary)
return "", fmt.Errorf("写入复用证据失败: %w", err)
}
if err := file.Sync(); err != nil {
file.Close()
os.Remove(temporary)
return "", fmt.Errorf("复用证据未落盘: %w", err)
}
if err := file.Close(); err != nil {
os.Remove(temporary)
return "", fmt.Errorf("复用证据未落盘: %w", err)
}
if err := os.Chmod(temporary, 0444); err != nil {
os.Remove(temporary)
return "", fmt.Errorf("复用证据只读化失败: %w", err)
}
if err := os.Rename(temporary, path); err != nil {
os.Remove(temporary)
return "", fmt.Errorf("复用证据落盘失败: %w", err)
}
return path, nil
}
// LoadReuseEvidence 取回并校验证据;摘要或绑定对不上就失败。
func LoadReuseEvidence(archiveRoot, digest string) (*ResourceReuseEvidence, error) {
if digest == "" {
return nil, errors.New("没有复用证据摘要")
}
raw, err := os.ReadFile(reusePath(archiveRoot, digest))
if err != nil {
return nil, fmt.Errorf("复用证据缺失(%s)", digest)
}
var evidence ResourceReuseEvidence
if err := json.Unmarshal(raw, &evidence); err != nil {
return nil, errors.New("复用证据不可解析")
}
if err := evidence.Verify(); err != nil {
return nil, err
}
if evidence.Digest != digest {
return nil, errors.New("复用证据摘要不符")
}
return &evidence, nil
}
// archivedRowByID 从归档部件里读出指定身份的行(原样,不改写)。
func archivedRowByID(archive *librarysnapshot.Archive, part librarysnapshot.Part, id string) (map[string]json.RawMessage, error) {
reader, err := archive.OpenPart(part)
if err != nil {
return nil, fmt.Errorf("读取归档 %s 失败", part.Name)
}
defer reader.Close()
decoder := json.NewDecoder(reader)
token, err := decoder.Token()
if err != nil || token != json.Delim('[') {
return nil, fmt.Errorf("归档 %s 不是行数组", part.Name)
}
for decoder.More() {
var row map[string]json.RawMessage
if err := decoder.Decode(&row); err != nil {
return nil, fmt.Errorf("解析归档 %s 行失败", part.Name)
}
var identity string
if json.Unmarshal(row["id"], &identity) == nil && identity == id {
return row, nil
}
}
return nil, nil
}
// targetResourceRow 读出目标里这一行;不存在返回 nil。
func (t *ProductionLibraryTarget) targetResourceRow(ctx context.Context, id string) (map[string]json.RawMessage, error) {
var raw map[string]json.RawMessage
err := t.pool.QueryRow(ctx, `SELECT to_jsonb(r) FROM resources r WHERE r.id=$1 AND r.tenant_id=$2`,
id, t.scope.TenantID).Scan(&raw)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
return nil, fmt.Errorf("读取目标资源 %s 失败", id)
}
return raw, nil
}
// cloneRow 复制一行,避免改写归档缓存/调用方数据。
func cloneRow(row map[string]json.RawMessage) map[string]json.RawMessage {
copied := make(map[string]json.RawMessage, len(row))
for key, value := range row {
copied[key] = value
}
return copied
}
// projectRow 只保留给定列集合里的列:目标行比归档多出的默认列不参与比对。
func projectRow(row map[string]json.RawMessage, columns map[string]bool) map[string]json.RawMessage {
projected := make(map[string]json.RawMessage, len(columns))
for key := range columns {
if value, ok := row[key]; ok {
projected[key] = value
}
}
return projected
}
// targetObjectSHA256 读取目标对象的实际字节并返回大小与摘要。
func (t *ProductionLibraryTarget) targetObjectSHA256(ctx context.Context, provider, path string) (int64, string, error) {
if t.config.ObjectsReader == nil {
return 0, "", errors.New("缺少目标对象读取器,无法证明对象字节一致")
}
reader, err := t.config.ObjectsReader.Open(ctx, provider, path)
if err != nil {
return 0, "", err
}
defer reader.Close()
hash := sha256.New()
size, err := io.Copy(hash, reader)
if err != nil {
return 0, "", err
}
return size, hex.EncodeToString(hash.Sum(nil)), nil
}
// resourceLocatorField 解析改写后的资源行里的定位符与提供方(与写入/校验同一口径)。
func resourceLocatorField(row map[string]json.RawMessage) (provider, path string, err error) {
if err := json.Unmarshal(row["provider"], &provider); err != nil {
return "", "", errors.New("资源行提供方不可解析")
}
if err := json.Unmarshal(row["physical_path"], &path); err != nil || path == "" {
return "", "", errors.New("资源行定位符不可解析")
}
return provider, path, nil
}
// provenanceFor 在"这个目标的持久来源"里找同一资源的同一行内容:
// - 成功同步过的快照(说明这个目标先前就把这行写进去过);
// - 这个目标自己的预备份(说明这行本来就是本目标历史状态的一部分)。
//
// 找不到就说明归属无法证明:即使目标里那行看起来一模一样,也不能覆盖/复用。
func provenanceFor(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, targetSnapshotID,
id, archivedDigest string, prior []libraryrelease.PriorSnapshot) (libraryrelease.PriorSnapshot, string, error) {
for _, candidate := range prior {
if candidate.JobID == "" || candidate.SnapshotID == "" {
continue
}
if candidate.SnapshotID == targetSnapshotID {
// 本次要写的快照本身就是一次成功同步写进目标的那一份。
return candidate, archivedDigest, nil
}
priorManifest, err := archive.Load(candidate.SnapshotID)
if err != nil {
// 来源快照不可读不能当作"没有来源":那会把不可审计变成静默放行。
return libraryrelease.PriorSnapshot{}, "", fmt.Errorf("复用来源快照 %s 不可读", candidate.SnapshotID)
}
if priorManifest.TenantID != manifest.TenantID || priorManifest.KnowledgeBaseID != manifest.KnowledgeBaseID {
continue
}
var part librarysnapshot.Part
found := false
for _, item := range priorManifest.Parts {
if item.Kind == "database" && item.Name == resourceReuseTableName {
part, found = item, true
break
}
}
if !found {
continue
}
row, err := archivedRowByID(archive, part, id)
if err != nil {
return libraryrelease.PriorSnapshot{}, "", err
}
if row == nil {
continue
}
digest, err := canonicalRowDigest(row)
if err != nil {
return libraryrelease.PriorSnapshot{}, "", err
}
if digest != archivedDigest {
continue
}
return candidate, digest, nil
}
return libraryrelease.PriorSnapshot{}, "", nil
}
// planReusableResource 是资源冲突的唯一放行路径:内容相同 + 字节相同 + 来源可审计。
// 返回写集合行(Existing + Reused)并把它登记到本次目标,写入阶段据此跳过这一行。
func (t *ProductionLibraryTarget) planReusableResource(ctx context.Context, archive *librarysnapshot.Archive,
manifest librarysnapshot.Manifest, part librarysnapshot.Part, id, owner string,
plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSetRow, error) {
archived, err := archivedRowByID(archive, part, id)
if err != nil {
return libraryrelease.WriteSetRow{}, err
}
if archived == nil {
return libraryrelease.WriteSetRow{}, fmt.Errorf("资源 %s 在快照 %s 的 resources 部件里找不到", id, plan.TargetSnapshotID)
}
archivedDigest, err := canonicalRowDigest(archived)
if err != nil {
return libraryrelease.WriteSetRow{}, err
}
columns := map[string]bool{}
for key := range archived {
columns[key] = true
}
retargeted := cloneRow(archived)
if err := t.retargetRow(resourceReuseTableName, retargeted); err != nil {
return libraryrelease.WriteSetRow{}, err
}
retargetedDigest, err := canonicalRowDigest(retargeted)
if err != nil {
return libraryrelease.WriteSetRow{}, err
}
existing, err := t.targetResourceRow(ctx, id)
if err != nil {
return libraryrelease.WriteSetRow{}, err
}
if existing == nil {
return libraryrelease.WriteSetRow{}, fmt.Errorf("资源 %s 在目标里已不存在,不能复用", id)
}
targetDigest, err := canonicalRowDigest(projectRow(existing, columns))
if err != nil {
return libraryrelease.WriteSetRow{}, err
}
if targetDigest != retargetedDigest {
return libraryrelease.WriteSetRow{}, fmt.Errorf(
"资源 ID 冲突:%s 已存在于目标租户但内容与本次快照不一致(目标 %s / 本次 %s),拒绝覆盖",
id, shortDigest(targetDigest), shortDigest(retargetedDigest))
}
provenance, provenanceDigest, err := provenanceFor(archive, manifest, plan.TargetSnapshotID, id, archivedDigest, plan.PriorSnapshots)
if err != nil {
return libraryrelease.WriteSetRow{}, err
}
if provenance.JobID == "" {
return libraryrelease.WriteSetRow{}, fmt.Errorf(
"资源 ID 冲突:%s 已存在于目标租户但不在预备份范围内,且没有可审计的持久来源(成功同步/预备份)证明它属于本目标,拒绝覆盖", id)
}
provider, path, err := resourceLocatorField(retargeted)
if err != nil {
return libraryrelease.WriteSetRow{}, err
}
objects := make([]reuseObject, 0, 2)
for _, item := range manifest.Parts {
if item.Kind != "resource" || item.Name != id {
continue
}
size, digest, err := t.targetObjectSHA256(ctx, provider, path)
if err != nil {
return libraryrelease.WriteSetRow{}, fmt.Errorf("资源 %s 的对象读不到,无法证明字节一致:%w", id, err)
}
if size != item.Bytes || digest != item.SHA256 {
return libraryrelease.WriteSetRow{}, fmt.Errorf(
"资源 ID 冲突:%s 的目标对象字节与本次快照不一致(%d/%d 字节),拒绝覆盖",
id, size, item.Bytes)
}
objects = append(objects, reuseObject{Name: item.Name, Path: path, Bytes: size, SHA256: digest})
}
evidence := &ResourceReuseEvidence{
Version: reuseEvidenceVersion, JobID: plan.JobID, Target: plan.Target,
TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID,
SourceTenantID: manifest.TenantID, SourceKnowledgeBaseID: manifest.KnowledgeBaseID,
TargetSnapshotID: plan.TargetSnapshotID, ResourceID: id,
ArchivedRowDigest: archivedDigest, RetargetedRowDigest: retargetedDigest, TargetRowDigest: targetDigest,
ProvenanceJobID: provenance.JobID, ProvenanceSnapshotID: provenance.SnapshotID, ProvenanceKind: provenance.Kind,
ProvenanceRowDigest: provenanceDigest, Objects: objects, CreatedAt: t.now().UTC(),
}
if _, err := PersistReuseEvidence(t.config.ArchiveRoot, evidence); err != nil {
return libraryrelease.WriteSetRow{}, fmt.Errorf("复用证据无法落盘,拒绝放行覆盖:%w", err)
}
if t.reusedResources == nil {
t.reusedResources = map[string]bool{}
}
t.reusedResources[id] = true
return libraryrelease.WriteSetRow{Table: resourceReuseTableName, ID: id, Existing: true,
Owner: owner, Reused: true, ReuseEvidence: evidence.Digest}, nil
}
func shortDigest(digest string) string {
if len(digest) > 12 {
return digest[:12]
}
return digest
}
@@ -0,0 +1,305 @@
package v071
import (
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryrelease"
)
// 这一组用例覆盖真实缺陷现场:同步 A 成功 → 手工恢复 B(对象保留、资源行不删)
// → 再同步 A。旧预检把"恢复保留下来的、本目标自己上次写进去的资源"判成
// "其它知识库的同 ID 冲突",整个同步在写第一笔之前被拒。
//
// 复用的放行条件必须是可审计的证据,而不是"目标里没人引用":
// - 目标现状那一行与本次要写的行逐列相同(除易变时间戳);
// - 该资源的对象字节与本次快照归档摘要一致;
// - 治理库里存在"这个目标成功同步过、且含有同一行内容"的快照/作业。
// 缺任何一条都必须继续拒绝,且拒绝发生在写入之前。
const (
wsHandleNew = "gggggggggggggggggggggg"
// resNewBody 是"A 的下一次发布"新增资源的内容;与 res-src 的 "12345678" 不同,
// 这样"本次真新建"和"证明可复用"在同一份写集合里能同时被观察到。
resNewBody = "87654321"
)
// reusePriors 是治理库会给出的来源清单:一次成功同步 + 它产生的预备份。
func reusePriors(syncJobID, syncSnapshotID string, prebackups ...string) []libraryrelease.PriorSnapshot {
priors := []libraryrelease.PriorSnapshot{{JobID: syncJobID, SnapshotID: syncSnapshotID, Kind: libraryrelease.PriorSnapshotSync}}
for _, snapshot := range prebackups {
priors = append(priors, libraryrelease.PriorSnapshot{JobID: syncJobID, SnapshotID: snapshot,
Kind: libraryrelease.PriorSnapshotPrebackup})
}
return priors
}
func requestForWithPrior(snapshotID, jobID string, prior []libraryrelease.PriorSnapshot) libraryrelease.Request {
request := requestFor(snapshotID)
request.JobID = jobID
request.PriorSnapshots = prior
return request
}
// syncThenRestore 造出现场:同步 A 成功,然后按生产恢复作业的方式恢复到 A 的
// 预备份(再走一次 Run:新的预备份 + 它自己的写集合)。恢复不删 resources 表,
// 所以 A 写进去的资源会作为"没人引用的行"留在目标里——正是本次缺陷的前提。
func syncThenRestore(t *testing.T, fx *writeSetFixture, snapshotID string) (*ProductionLibraryTarget,
*ProductionBackupStore, libraryrelease.Outcome, []libraryrelease.PriorSnapshot) {
t.Helper()
target := fx.target(t)
backups := fx.backups(t, target)
first := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a", nil), target, backups, nil)
if first.State != "succeeded" {
t.Fatalf("首次同步 A 必须成功:%+v", first)
}
if first.Backup.WriteSet == nil {
t.Fatal("首次同步必须落盘写集合")
}
// 恢复作业同样只认已登记来源:它自己刚拍的那份预备份就是本目标的历史状态。
restore := libraryrelease.Run(fx.ctx, requestForWithPrior(first.Backup.SnapshotID, "job-restore-b",
reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)), target, backups, nil)
if restore.State != "succeeded" {
t.Fatalf("手工恢复 B 必须成功:%+v", restore)
}
var bindings int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resource_bindings WHERE resource_id='res-src'`).Scan(&bindings); err != nil {
t.Fatal(err)
}
if bindings != 0 {
t.Fatalf("恢复后 res-src 不应还有绑定行(现场前提):%d", bindings)
}
return target, backups, first, reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)
}
// resourceFingerprint 取 PG 行的 xmin 与整行 JSON:xmin 不变才能证明"没有重写这一行"。
func resourceFingerprint(t *testing.T, pool *pgxpool.Pool, id string) (string, string) {
t.Helper()
var xmin, row string
if err := pool.QueryRow(t.Context(), `SELECT xmin::text, to_jsonb(r)::text FROM resources r WHERE id=$1`, id).
Scan(&xmin, &row); err != nil {
t.Fatalf("读取资源 %s: %v", id, err)
}
return xmin, row
}
func resourceRowOf(t *testing.T, pool *pgxpool.Pool, id string) string {
t.Helper()
var row string
if err := pool.QueryRow(t.Context(), `SELECT to_jsonb(r)::text FROM resources r WHERE id=$1`, id).Scan(&row); err != nil {
t.Fatalf("读取资源 %s: %v", id, err)
}
return row
}
func TestLibraryResyncAfterRestoreReusesProvenResourcesIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, first, priors := syncThenRestore(t, fx, snapshotID)
beforeXmin, beforeRow := resourceFingerprint(t, fx.pool, "res-src")
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
otherBefore := resourceRowOf(t, fx.pool, "res-other")
sharedBefore := resourceRowOf(t, fx.pool, "res-shared")
foreignBefore := resourceRowOf(t, fx.pool, "res-foreign")
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a-again", priors),
target, backups, nil)
if outcome.State != "succeeded" {
t.Fatalf("恢复后再次同步 A 必须完成:%+v", outcome)
}
if outcome.Backup.WriteSet == nil {
t.Fatal("写集合必须落盘")
}
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
if err != nil {
t.Fatalf("写集合记录取不回来:%v", err)
}
reused := ledger.ReusedIDs("resources")
if len(reused) != 1 || reused[0] != "res-src" {
t.Fatalf("复用资源 = %v,want [res-src]", reused)
}
if created := ledger.CreatedIDs("resources"); len(created) != 0 {
t.Fatalf("复用资源不能被记成本次新建(恢复会删掉它):%v", created)
}
var reuseRow libraryrelease.WriteSetRow
for _, row := range ledger.Rows {
if row.Table == "resources" && row.ID == "res-src" {
reuseRow = row
}
}
if !reuseRow.Existing || !reuseRow.Reused || reuseRow.ReuseEvidence == "" {
t.Fatalf("复用行必须带证据摘要并标记为写前已存在:%+v", reuseRow)
}
evidence, err := LoadReuseEvidence(fx.archiveRoot, reuseRow.ReuseEvidence)
if err != nil {
t.Fatalf("复用证据取不回来:%v", err)
}
if evidence.ProvenanceSnapshotID != snapshotID || evidence.ProvenanceJobID != first.Backup.WriteSet.JobID {
t.Fatalf("复用证据没有指向前一次成功同步:%+v", evidence)
}
if evidence.TargetRowDigest != evidence.RetargetedRowDigest || evidence.ProvenanceRowDigest != evidence.ArchivedRowDigest {
t.Fatalf("复用证据的行摘要不一致:%+v", evidence)
}
if len(evidence.Objects) != 1 || evidence.Objects[0].SHA256 == "" {
t.Fatalf("复用证据必须逐个记录对象字节:%+v", evidence.Objects)
}
// 复用不许改原行:xmin 与整行 JSON 都必须原样。
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
if afterXmin != beforeXmin {
t.Fatalf("复用资源被重写了:xmin %s -> %s", beforeXmin, afterXmin)
}
if afterRow != beforeRow {
t.Fatalf("复用资源内容变了:\n%s\n->\n%s", beforeRow, afterRow)
}
// 历史孤儿/其它知识库/共享/外租户资源一律不许被覆盖。
for id, want := range map[string]string{"res-orphan": orphanBefore, "res-other": otherBefore,
"res-shared": sharedBefore, "res-foreign": foreignBefore} {
if got := resourceRowOf(t, fx.pool, id); got != want {
t.Fatalf("资源 %s 被改动:\n%s\n->\n%s", id, want, got)
}
}
}
func TestLibraryResyncAfterRestoreRejectsReuseWithoutProvenanceIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, first, _ := syncThenRestore(t, fx, snapshotID)
before := universeDump(t, fx.pool)
// 已登记的来源里没有 res-src(只有这次同步之前的预备份):即使目标那一行
// 看起来一模一样,也不许放行。
noSource := []libraryrelease.PriorSnapshot{{JobID: first.Backup.WriteSet.JobID,
SnapshotID: first.Backup.SnapshotID, Kind: libraryrelease.PriorSnapshotPrebackup}}
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-no-provenance", noSource), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("没有来源证据时必须拒绝:%+v", outcome)
}
if outcome.FailedAt != "" {
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at=%s", outcome.FailedAt)
}
if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "可审计") {
t.Fatalf("拒绝原因必须点名资源并说明缺来源证据:%s", outcome.Error)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
}
}
func TestLibraryResyncAfterRestoreRejectsReuseWhenContentDiffersIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
// 同 ID 同租户,但内容被改过(这里改大小,等价于"另一份文件占了这个 ID")。
if _, err := fx.pool.Exec(fx.ctx, `UPDATE resources SET size=size+1 WHERE id='res-src'`); err != nil {
t.Fatal(err)
}
changed := resourceRowOf(t, fx.pool, "res-src")
before := universeDump(t, fx.pool)
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-content-diff", priors), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("内容不一致必须拒绝:%+v", outcome)
}
if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "内容") {
t.Fatalf("拒绝原因必须点名内容不一致:%s", outcome.Error)
}
if resourceRowOf(t, fx.pool, "res-src") != changed {
t.Fatal("被拒绝的同步不许改目标资源")
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
}
}
// 同租户、同 ID、同内容,但来源快照属于另一个知识库:仍然不许复用。
// "归属无法证明"包括"来源不同",不能因为字节一样就把别人的行当成自己的。
func TestLibraryReuseRequiresSameSourceScopeIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
otherSource := sealSourceSnapshotScope(t, fx, wsSourceTenant+1, "kb-source-other", nil, nil)
before := universeDump(t, fx.pool)
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(otherSource, "job-sync-other-source", priors), target, backups, nil)
if outcome.State != "failed" {
t.Fatalf("来源范围不同必须拒绝复用:%+v", outcome)
}
if !strings.Contains(outcome.Error, "可审计") {
t.Fatalf("拒绝原因必须说明来源不可审计:%s", outcome.Error)
}
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
}
}
// 部分写失败:database/resources 已提交、wiki 因重复 slug 失败 → 回滚只清"本次真新建",
// 复用资源与历史孤儿原内容完整保留(xmin 也不变)。
func TestLibraryHalfWriteRollbackKeepsReusedResourcesIsolated(t *testing.T) {
fx := openWriteSetFixture(t)
seedTargetData(t, fx)
snapshotID := sealSourceSnapshot(t, fx, nil)
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
reuseXmin, reuseRow := resourceFingerprint(t, fx.pool, "res-src")
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
next := sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB,
func(parts map[string][]map[string]any) {
parts["chunks"] = append(parts["chunks"], map[string]any{"id": "chunk-src-2", "tenant_id": wsSourceTenant,
"knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src", "content": "新增正文 resource://" + wsHandleNew})
parts["resources"] = append(parts["resources"], map[string]any{"id": "res-new", "handle": wsHandleNew,
"tenant_id": wsSourceTenant, "storage_backend_id": "backend-source", "provider": "oss",
"physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-new"),
"size": int64(len(resNewBody))})
parts["resource_bindings"] = append(parts["resource_bindings"], map[string]any{"id": "bind-src-2",
"resource_id": "res-new", "tenant_id": wsSourceTenant, "owner_type": "chunk",
"owner_id": "chunk-src-2", "relation": "attachment"})
// 同一作用域两页同 slug:wiki 的唯一索引让 wiki 阶段失败(真实半写现场)。
parts["wiki_pages"] = []map[string]any{
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
}
}, map[string]string{"res-new": resNewBody})
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(next, "job-sync-half-write", priors), target, backups, nil)
if outcome.State != "rolled_back" {
t.Fatalf("wiki 半写必须回滚:%+v", outcome)
}
if outcome.FailedAt != libraryrelease.StageWiki {
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
}
ledger := outcome.Backup.WriteSet
if ledger == nil {
t.Fatal("半写作业必须留下写集合")
}
if created := ledger.CreatedIDs("resources"); len(created) != 1 || created[0] != "res-new" {
t.Fatalf("本次真新建的资源 = %v,want [res-new]", created)
}
if reused := ledger.ReusedIDs("resources"); len(reused) != 1 || reused[0] != "res-src" {
t.Fatalf("复用资源 = %v,want [res-src]", reused)
}
// 本次真新建的清干净。
var residue int64
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-new'`).Scan(&residue); err != nil {
t.Fatal(err)
}
if residue != 0 {
t.Fatal("回滚后本次新建的资源必须被清理")
}
// 复用资源与历史孤儿原内容(含 xmin)保留。
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
if afterXmin != reuseXmin || afterRow != reuseRow {
t.Fatalf("回滚动了复用资源:xmin %s->%s\n%s\n->\n%s", reuseXmin, afterXmin, reuseRow, afterRow)
}
if resourceRowOf(t, fx.pool, "res-orphan") != orphanBefore {
t.Fatal("回滚动了历史孤儿资源")
}
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
t.Fatalf("回滚必须与预备份一致:%v", err)
}
}
@@ -132,6 +132,14 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library
if len(plan.Universe) == 0 {
return libraryrelease.WriteSet{}, errors.New("预备份没有整实例内容证据,拒绝在没有保护证据的情况下写目标")
}
// 资源复用的"整行内容"必须按写入时的同一套范围/存储改写来比,所以先把目标
// 存储后端解析出来;解析不到就等于写不出可解析的定位符,直接拒绝。
if err := t.resolveStorageBackend(ctx); err != nil {
return libraryrelease.WriteSet{}, err
}
// 每次预检都从空集开始:同一进程里复用目标对象时,上一次作业的"可复用"
// 判断不能泄漏到这一次。
t.reusedResources = map[string]bool{}
archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
if err != nil {
return libraryrelease.WriteSet{}, errors.New("归档不可用")
@@ -173,6 +181,20 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id})
continue
}
if part.Name == resourceReuseTableName && !backupResources[id] {
// 资源不在预备份范围,但可能正是这个目标上一次成功同步写进去、
// 又被恢复保留下来的那份内容。只有"内容/字节相同 + 来源可审计"
// 才允许复用(不重写、不清理);否则按跨范围冲突拒绝。
if err := t.assertResourceTenant(id, owner); err != nil {
return libraryrelease.WriteSet{}, err
}
row, reuseErr := t.planReusableResource(ctx, archive, manifest, part, id, owner, plan)
if reuseErr != nil {
return libraryrelease.WriteSet{}, reuseErr
}
set.Rows = append(set.Rows, row)
continue
}
if err := t.assertWriteSetOwnership(part.Name, id, owner, backupResources); err != nil {
return libraryrelease.WriteSet{}, err
}
@@ -265,6 +287,15 @@ WHERE b.id = ANY($1::text[])`
return owners, rows.Err()
}
// assertResourceTenant 只检查"同 ID 是否属于别的租户":resources 没有
// knowledge_base_id,同租户不等于同知识库,范围问题交给后续的复用证明。
func (t *ProductionLibraryTarget) assertResourceTenant(id, owner string) error {
if owner != fmt.Sprintf("%d/", t.scope.TenantID) {
return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner)
}
return nil
}
// assertWriteSetOwnership 是跨范围 ID 冲突的预检:同 ID 已经是别人的行时,
// 写入会改掉别人的数据(resources 尤其危险:ON CONFLICT 会整行覆盖)。
// 宁可整次拒绝,也不许"覆盖一下看起来也能跑"。
@@ -272,8 +303,8 @@ func (t *ProductionLibraryTarget) assertWriteSetOwnership(table, id, owner strin
target := fmt.Sprintf("%d/%s", t.scope.TenantID, t.scope.KnowledgeBaseID)
if table == "resources" {
// 资源只有租户:同租户不等于同知识库,必须落在预备份范围内才算自己的。
if owner != fmt.Sprintf("%d/", t.scope.TenantID) {
return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner)
if err := t.assertResourceTenant(id, owner); err != nil {
return err
}
if !backupResources[id] {
return fmt.Errorf("资源 ID 冲突:%s 存在于目标租户但不在预备份范围内(属于其它知识库),拒绝覆盖", id)
@@ -293,6 +293,14 @@ func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb,
// sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。
func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string {
t.Helper()
return sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB, tweak, nil)
}
// sealSourceSnapshotScope 允许指定来源范围/附加对象:来源快照的身份决定
// "同一资源行是否属于同一个来源",复用证据必须按它区分。
func sealSourceSnapshotScope(t *testing.T, fx *writeSetFixture, sourceTenant uint64, sourceKB string,
tweak func(parts map[string][]map[string]any), extraObjects map[string]string) string {
t.Helper()
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
if err != nil {
@@ -307,20 +315,20 @@ func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[
return encoded
}
parts := map[string][]map[string]any{
"knowledges": {{"id": "doc-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
"knowledges": {{"id": "doc-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
"parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}},
"chunks": {{"id": "chunk-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
"chunks": {{"id": "chunk-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
"knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}},
"knowledge_tags": {},
"knowledge_tag_relations": {},
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": wsSourceTenant,
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": sourceTenant,
"storage_backend_id": "backend-source", "provider": "oss",
"physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-src"),
"physical_path": wsLocator("backend-source", "oss", "source-bucket", sourceTenant, "res-src"),
"size": int64(8)}},
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": wsSourceTenant,
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": sourceTenant,
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}},
"wiki_folders": {},
"wiki_pages": {{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"}},
"wiki_pages": {{"id": "wiki-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "slug": "index", "content": "来源 wiki"}},
"wiki_page_issues": {},
"wiki_log_entries": {},
}
@@ -350,17 +358,38 @@ func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[
}
resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1
manifestParts = append(manifestParts, resourcePart)
indexPart, err := archive.Put(bytes.NewReader(encode([]map[string]any{{"id": "chunk-src", "source": map[string]any{
"tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src",
"chunk_id": "chunk-src", "source_id": "chunk-src", "content": "来源正文 resource://" + wsHandleSource,
"embedding": []float64{0.1, 0.2}}}})))
extraNames := make([]string, 0, len(extraObjects))
for name := range extraObjects {
extraNames = append(extraNames, name)
}
sort.Strings(extraNames)
for _, name := range extraNames {
part, err := archive.Put(bytes.NewReader([]byte(extraObjects[name])))
if err != nil {
t.Fatal(err)
}
part.Kind, part.Name, part.Count = "resource", name, 1
manifestParts = append(manifestParts, part)
}
// 索引文档按快照里的分块逐条生成:多一个分块就必须多一条向量,否则
// 校验会把"写入正确"判成索引缺文档。
indexRows := make([]map[string]any, 0, len(parts["chunks"]))
for _, chunk := range parts["chunks"] {
id, _ := chunk["id"].(string)
knowledge, _ := chunk["knowledge_id"].(string)
content, _ := chunk["content"].(string)
indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{
"tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "knowledge_id": knowledge,
"chunk_id": id, "source_id": id, "content": content, "embedding": []float64{0.1, 0.2}}})
}
indexPart, err := archive.Put(bytes.NewReader(encode(indexRows)))
if err != nil {
t.Fatal(err)
}
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", 1
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows))
manifestParts = append(manifestParts, indexPart)
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
Format: librarysnapshot.Format, Scope: "internal", TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB,
Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB,
CreatedAt: time.Now().UTC(), Parts: manifestParts,
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",