diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index 9c23860..4c7b7b0 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -69,7 +69,13 @@ func (t *contentTrackedTarget) PlanWriteSet(ctx context.Context, plan libraryrel if !ok { return libraryrelease.WriteSet{}, errors.New("目标实现不支持写集合预检:拒绝在没有可追溯写集合的情况下写生产") } - return planner.PlanWriteSet(ctx, plan) + set, err := planner.PlanWriteSet(ctx, plan) + if err == nil { + // 日志是"wrapper 路径真的执行了预检"的现场证据:接管排查先看这一行。 + log.Printf("library writeset planned job=%s target=%s digest=%s rows=%d reused=%d target_snapshot=%s", + set.JobID, set.Target, set.Digest, len(set.Rows), len(set.ReusedIDs("resources")), set.TargetSnapshotID) + } + return set, err } // libraryReleasePayload 是同步与恢复共用的队列载荷形状。 @@ -157,6 +163,17 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a if err != nil || !found { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "没有与目标快照匹配的有效发布授权", "prebackup") } + // 资源复用的来源证据:只认这个目标"成功同步过且登记过生效版本"的快照与它自己 + // 的预备份。读不到就按空处理——那只会让复用被拒绝,绝不会误放行覆盖。 + priorSnapshots := make([]libraryrelease.PriorSnapshot, 0, 8) + if sources, sourceErr := store.LibraryReuseSources(ctx, payload.Scope, 100); sourceErr != nil { + log.Printf("library %s prior reuse sources unavailable: %v", payload.LibraryJobID, sourceErr) + } else { + for _, source := range sources { + priorSnapshots = append(priorSnapshots, libraryrelease.PriorSnapshot{JobID: source.JobID, + SnapshotID: source.SnapshotID, Kind: source.Kind, FinishedAt: source.FinishedAt}) + } + } // 生产索引映射基线:优先当前生效版本,其次任何一次成功预备份(它也记录了生产实际映射)。 // 只认"生效版本"会在恢复把版本 supersede 掉之后误判"索引被改动"。 knownMapping := "" @@ -325,6 +342,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope, SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now, + PriorSnapshots: priorSnapshots, Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID, ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}} _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now) diff --git a/cmd/worker/library_writeset.go b/cmd/worker/library_writeset.go index ae22794..f6c3013 100644 --- a/cmd/worker/library_writeset.go +++ b/cmd/worker/library_writeset.go @@ -84,6 +84,7 @@ func writeSetEvidence(existing map[string]any, set libraryrelease.WriteSet, mapp evidence["writeset_universe_digest"] = set.UniverseDigest() evidence["writeset_target_snapshot"] = set.TargetSnapshotID evidence["writeset_rows"] = len(set.Rows) + evidence["writeset_reused_rows"] = len(set.ReusedIDs("resources")) return evidence } diff --git a/cmd/worker/library_writeset_test.go b/cmd/worker/library_writeset_test.go index 86dc808..74b46fc 100644 --- a/cmd/worker/library_writeset_test.go +++ b/cmd/worker/library_writeset_test.go @@ -167,10 +167,12 @@ func (t *plainTarget) Restore(context.Context, libraryrelease.BackupRef) error type planningTarget struct { plainTarget planned int + last libraryrelease.WriteSetPlan } -func (t *planningTarget) PlanWriteSet(context.Context, libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) { +func (t *planningTarget) PlanWriteSet(_ context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) { t.planned++ + t.last = plan return libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: "job-1", Target: "internal"}, nil } @@ -181,12 +183,17 @@ func TestContentTrackedTargetForwardsWriteSetPlanning(t *testing.T) { if !ok { t.Fatal("包装后的目标必须仍然实现 WriteSetPlanner,否则写集合会被静默跳过") } - if _, err := planner.PlanWriteSet(context.Background(), libraryrelease.WriteSetPlan{JobID: "job-1", Target: "internal"}); err != nil { + if _, err := planner.PlanWriteSet(context.Background(), libraryrelease.WriteSetPlan{JobID: "job-1", Target: "internal", + PriorSnapshots: []libraryrelease.PriorSnapshot{{JobID: "job-0", SnapshotID: "snap-0", Kind: libraryrelease.PriorSnapshotSync}}}); err != nil { t.Fatalf("plan: %v", err) } if inner.planned != 1 { t.Fatalf("预检必须落到内层目标,planned=%d", inner.planned) } + // 来源证据也必须透传:丢掉它就会让"恢复保留下来的资源"永远无法证明可复用。 + if len(inner.last.PriorSnapshots) != 1 || inner.last.PriorSnapshots[0].SnapshotID != "snap-0" { + t.Fatalf("PriorSyncs 没有透传到内层:%+v", inner.last) + } // 内层不支持预检 ⇒ 硬错误,不许"没有台账也照写"。 strict := &contentTrackedTarget{inner: &plainTarget{}, onApply: func() {}} diff --git a/internal/libraryrelease/release.go b/internal/libraryrelease/release.go index 0de15be..1a56e73 100644 --- a/internal/libraryrelease/release.go +++ b/internal/libraryrelease/release.go @@ -45,6 +45,10 @@ type Request struct { ParamsDigest string Grant Grant Now time.Time + // PriorSnapshots 是这个目标的持久来源(成功同步过的快照 + 自己的预备份), + // 由治理库提供。资源复用只认它作证据;为空时"目标里已有但不在预备份范围" + // 的资源一律按冲突拒绝。 + PriorSnapshots []PriorSnapshot } // CurrentState 是生产当前整库的指纹,用于预备份与最终验证。 @@ -213,7 +217,8 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor if backup.WriteSet == nil { if planner, ok := target.(WriteSetPlanner); ok { set, planErr := planner.PlanWriteSet(ctx, WriteSetPlan{JobID: request.JobID, Target: request.Target, - BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: request.SnapshotID, Universe: backup.Universe}) + BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: request.SnapshotID, Universe: backup.Universe, + PriorSnapshots: request.PriorSnapshots}) if planErr != nil { outcome.State = "failed" outcome.Error = "写集合/冲突预检失败,未对生产执行任何写入:" + planErr.Error() diff --git a/internal/libraryrelease/writeset.go b/internal/libraryrelease/writeset.go index 89c1ba1..c7345b0 100644 --- a/internal/libraryrelease/writeset.go +++ b/internal/libraryrelease/writeset.go @@ -28,6 +28,16 @@ type WriteSetRow struct { Existing bool `json:"existing"` // Owner 是写前该行的归属("/"),不存在时为空。 Owner string `json:"owner,omitempty"` + // Reused 表示这一行虽然写前就在目标里、却不在预备份范围内,但已经用 + // "持久作业/快照证据 + 整行内容 + 对象字节"证明与本次要写的内容完全一致, + // 因此本次既不重写它、也不在恢复时清理它。 + // + // 不能只因为"它没被任何绑定/正文引用"就当成自己的:历史孤儿、同租户另一个 + // 知识库暂未绑定的资源同样满足"没人引用",覆盖或删除同样是数据丢失。 + Reused bool `json:"reused,omitempty"` + // ReuseEvidence 是可复用证据文档的摘要(reuse/.json),独立于写集合, + // 防止把这次推断伪装成历史台账。 + ReuseEvidence string `json:"reuse_evidence,omitempty"` } // WriteSetPlan 是调用方在写第一笔之前交给目标的信息:绑定作业、目标、预备份与快照, @@ -38,6 +48,29 @@ type WriteSetPlan struct { BackupSnapshotID string TargetSnapshotID string Universe map[string]TableFingerprint + // PriorSnapshots 是治理库里"这个目标的持久来源":成功同步过的快照,以及这个 + // 目标自己的预备份。资源复用必须沿它把来源追到真实登记过的快照,而不是按 + // "目标里没人引用"推断归属。 + PriorSnapshots []PriorSnapshot +} + +// 复用来源的两种合法类别。 +const ( + // PriorSnapshotSync:这个目标成功同步过这份快照(且登记了生效版本)。 + PriorSnapshotSync = "sync" + // PriorSnapshotPrebackup:这份快照是从这个目标自己身上拍的预备份,把它写回去 + // 就是回到本目标的历史状态。 + PriorSnapshotPrebackup = "prebackup" +) + +// PriorSnapshot 是一次**已登记**的来源:作业身份 + 快照身份 + 来源类别。 +// 只有它也含有同一个资源 ID、且行内容完全一致,才允许把目标里那行当作 +// "本次写入的等价物"。 +type PriorSnapshot struct { + JobID string `json:"job_id"` + SnapshotID string `json:"snapshot_id"` + Kind string `json:"kind"` + FinishedAt time.Time `json:"finished_at,omitempty"` } // WriteSet 是"这一次发布的写集合 + 写前整实例内容证据",内容寻址、可校验、 @@ -124,6 +157,14 @@ func (w *WriteSet) Verify() error { if w.Digest == "" { return errors.New("写集合记录没有摘要") } + for _, row := range w.Rows { + if row.Reused && !row.Existing { + return fmt.Errorf("写集合记录把 %s/%s 标成可复用却不是写前已存在行", row.Table, row.ID) + } + if row.Reused && row.ReuseEvidence == "" { + return fmt.Errorf("写集合记录把 %s/%s 标成可复用却没有证据摘要", row.Table, row.ID) + } + } digest, err := w.computeDigest() if err != nil { return err @@ -158,6 +199,19 @@ func (w WriteSet) ExistingIDs(table string) []string { return ids } +// ReusedIDs 返回本次"证明可复用、因此既不重写也不清理"的精确身份。 +// 它与 CreatedIDs 互斥,恢复清理一律不看这里(复用行必须原样保留)。 +func (w WriteSet) ReusedIDs(table string) []string { + ids := make([]string, 0, 8) + for _, row := range w.Rows { + if row.Table == table && row.Reused { + ids = append(ids, row.ID) + } + } + sort.Strings(ids) + return ids +} + // UniverseDigest 是整实例证据的摘要,随台账一起登记,便于独立核对。 func (w WriteSet) UniverseDigest() string { if len(w.Universe) == 0 { diff --git a/internal/libraryrelease/writeset_test.go b/internal/libraryrelease/writeset_test.go index 1a01d28..e794308 100644 --- a/internal/libraryrelease/writeset_test.go +++ b/internal/libraryrelease/writeset_test.go @@ -102,3 +102,65 @@ func TestWriteSetCreatedIDsOnlyCoversRowsCreatedByThisRelease(t *testing.T) { t.Fatal("universe digest must be derived from the whole-instance evidence") } } + +// 复用行是"写前已存在 + 有证据摘要"的一种:既不算本次新建(恢复不许删), +// 也不能没有证据摘要,更不能被标成新建。 +func TestWriteSetReusedRowsStayOutOfCleanupAndRequireEvidence(t *testing.T) { + set := WriteSet{Version: WriteSetVersion, JobID: "job-2", Target: "internal", TenantID: 10001, + KnowledgeBaseID: "kb", BackupSnapshotID: "backup-2", TargetSnapshotID: "snap-2", + CreatedAt: time.Date(2026, 10, 30, 1, 0, 0, 0, time.UTC), + Rows: []WriteSetRow{ + {Table: "resources", ID: "res-reused", Existing: true, Owner: "10001/", Reused: true, ReuseEvidence: "digest-1"}, + {Table: "resources", ID: "res-new"}, + }, + Universe: map[string]TableFingerprint{"resources": {Count: 3, Digest: "abc"}}} + if err := set.Seal(); err != nil { + t.Fatal(err) + } + if err := set.Verify(); err != nil { + t.Fatalf("复用行必须能通过校验: %v", err) + } + if created := set.CreatedIDs("resources"); len(created) != 1 || created[0] != "res-new" { + t.Fatalf("复用行被当成本次新建(恢复会删掉它):%v", created) + } + if reused := set.ReusedIDs("resources"); len(reused) != 1 || reused[0] != "res-reused" { + t.Fatalf("reused resources = %v", reused) + } + + noEvidence := set + noEvidence.Rows = append([]WriteSetRow{}, set.Rows...) + for index := range noEvidence.Rows { + if noEvidence.Rows[index].ID == "res-reused" { + noEvidence.Rows[index].ReuseEvidence = "" + } + } + if err := noEvidence.Seal(); err != nil { + t.Fatal(err) + } + if err := noEvidence.Verify(); err == nil { + t.Fatal("复用行没有证据摘要时必须校验失败") + } + + notExisting := set + notExisting.Rows = append([]WriteSetRow{}, set.Rows...) + for index := range notExisting.Rows { + if notExisting.Rows[index].ID == "res-reused" { + notExisting.Rows[index].Existing = false + } + } + if err := notExisting.Seal(); err != nil { + t.Fatal(err) + } + if err := notExisting.Verify(); err == nil { + t.Fatal("复用行必须同时是写前已存在行(否则会被恢复当残渣)") + } + + // 旧记录(没有复用字段)仍然必须能读回:格式演进不能把历史台账判成损坏。 + legacy := sampleWriteSet() + if err := legacy.Seal(); err != nil { + t.Fatal(err) + } + if err := legacy.Verify(); err != nil { + t.Fatalf("历史写集合记录必须继续有效: %v", err) + } +} diff --git a/internal/platform/postgres/library_jobs.go b/internal/platform/postgres/library_jobs.go index 66ad5fb..ad4ff36 100644 --- a/internal/platform/postgres/library_jobs.go +++ b/internal/platform/postgres/library_jobs.go @@ -102,6 +102,50 @@ FROM library_jobs WHERE id=$1`, id).Scan(&job.ID, &job.Scope, &kind, &state, &jo return job, nil } +// LibraryReuseSource 是一条"这个目标的持久来源"记录:成功同步过的快照, +// 或这个目标自己的预备份。资源复用只认它作来源证据。 +type LibraryReuseSource struct { + JobID string + SnapshotID string + Kind string // sync | prebackup + FinishedAt time.Time +} + +// LibraryReuseSources 读这个目标的持久来源(含已被恢复顶替的生效版本)。 +// 同步来源要求同时存在 production_effective 版本登记:只写成功但版本没登记的 +// 作业不算可审计来源。 +func (s *Store) LibraryReuseSources(ctx context.Context, scope string, limit int) ([]LibraryReuseSource, error) { + if limit < 1 || limit > 200 { + limit = 100 + } + rows, err := s.db.Query(ctx, ` +SELECT j.id, j.target_snapshot_id, 'sync', coalesce(j.finished_at, j.updated_at) +FROM library_jobs j +WHERE j.scope=$1 AND j.kind='sync' AND j.state='succeeded' AND coalesce(j.target_snapshot_id,'')<>'' + AND EXISTS (SELECT 1 FROM library_versions v + WHERE v.scope=j.scope AND v.kind='production_effective' + AND v.snapshot_id=j.target_snapshot_id AND v.source_job_id=j.id) +UNION ALL +SELECT coalesce(v.source_job_id,''), v.snapshot_id, 'prebackup', v.created_at +FROM library_versions v +WHERE v.scope=$1 AND v.kind='production_prebackup' AND coalesce(v.snapshot_id,'')<>'' +ORDER BY 4 DESC +LIMIT $2`, scope, limit) + if err != nil { + return nil, fmt.Errorf("load library reuse sources: %w", err) + } + defer rows.Close() + sources := make([]LibraryReuseSource, 0, 8) + for rows.Next() { + var source LibraryReuseSource + if err := rows.Scan(&source.JobID, &source.SnapshotID, &source.Kind, &source.FinishedAt); err != nil { + return nil, fmt.Errorf("scan library reuse source: %w", err) + } + sources = append(sources, source) + } + return sources, rows.Err() +} + func (s *Store) libraryStages(ctx context.Context, id string) ([]governance.LibraryStage, error) { rows, err := s.db.Query(ctx, ` SELECT stage, state, total, completed, bytes_total, bytes_done, detail, updated_at diff --git a/internal/releaseadapter/v071/library_production_target.go b/internal/releaseadapter/v071/library_production_target.go index d4eb824..7dd3ab3 100644 --- a/internal/releaseadapter/v071/library_production_target.go +++ b/internal/releaseadapter/v071/library_production_target.go @@ -94,6 +94,9 @@ type ProductionLibraryTarget struct { snapshotKB string // targetEmbeddingModel 是目标知识库绑定的 embedding 模型 id(文档行要换成它)。 targetEmbeddingModel string + // reusedResources 是"写前已存在、且已证明可复用"的资源身份:写入阶段跳过它们, + // 既不重写行也不重传对象。PlanWriteSet 与 Apply 在同一个作业里串行执行。 + reusedResources map[string]bool } // resolveStorageBackend 在目标库里找本次写入的存储后端。找不到就报错:宁可不写, @@ -383,6 +386,14 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc if len(row) == 0 { continue } + // 已证明与目标现状完全一致的资源:不重写这一行(连 updated_at 都不碰)。 + // 它在写集合里是 Existing+Reused,恢复清理也不会看它。 + if table == resourceReuseTableName && t.reusedResources != nil { + var identity string + if json.Unmarshal(row["id"], &identity) == nil && t.reusedResources[identity] { + continue + } + } // 行必须先映射到目标范围再落库:原样插入会把行写进来源租户/知识库, // 目标范围的删除、校验与恢复都看不见它们。 if err := t.retargetRow(table, row); err != nil { @@ -465,6 +476,11 @@ func (t *ProductionLibraryTarget) importResources(ctx context.Context, archive * if !ok || location.Path == "" { return fmt.Errorf("生产缺少资源 %s 的存储路径", part.Name) } + // 复用资源的对象字节已经在计划阶段逐字节比过:不再重传(重传只会改 + // 对象的写入时间,没有任何收益)。 + if t.reusedResources[part.Name] { + continue + } reader, err := archive.OpenPart(part) if err != nil { return err diff --git a/internal/releaseadapter/v071/library_reuse.go b/internal/releaseadapter/v071/library_reuse.go new file mode 100644 index 0000000..f20e808 --- /dev/null +++ b/internal/releaseadapter/v071/library_reuse.go @@ -0,0 +1,444 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +// 资源复用证据:目标里已经有这个资源 ID、但它不在预备份范围内时,唯一安全的 +// 处理是证明"本次要写的那一行/那一份对象字节,已经原样存在于目标里"。 +// +// 为什么不能按"没人引用"推断归属:历史孤儿、同租户另一个知识库暂未绑定的资源 +// 同样没人引用;按它放行等于允许覆盖别人的数据。这里的证据必须同时满足三件事: +// 1. 有可审计的持久来源(治理库里这个目标成功同步过的快照或它自己的预备份, +// 且同一资源行内容一致); +// 2. 整行内容(除易变时间戳外)与本次写入将要落地的行完全一致; +// 3. 该资源的对象字节与本次快照归档摘要一致。 +// +// 证据独立落盘(reuse/.json),摘要写进写集合行;它不冒充历史写集合—— +// 那两次没有 writeset 的成功作业依然没有 writeset。 + +const ( + reuseDirName = "reuse" + reuseEvidenceVersion = 1 + resourceReuseTableName = "resources" +) + +// reuseObject 是一个资源对象的字节证据:归档摘要 + 目标现状必须匹配。 +type reuseObject struct { + Name string `json:"name"` + Path string `json:"path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +// ResourceReuseEvidence 是"这一行可以复用"的完整证据文档。 +// Digest 覆盖除自己以外的全部内容:被改动就对不上写集合行里的摘要。 +type ResourceReuseEvidence struct { + Version int `json:"version"` + JobID string `json:"job_id"` + Target string `json:"target"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + SourceTenantID uint64 `json:"source_tenant_id"` + SourceKnowledgeBaseID string `json:"source_knowledge_base_id"` + TargetSnapshotID string `json:"target_snapshot_id"` + ResourceID string `json:"resource_id"` + // ArchivedRowDigest 是来源快照里那一行(不含易变时间戳)的摘要。 + ArchivedRowDigest string `json:"archived_row_digest"` + // RetargetedRowDigest 是同一行按本次目标范围/存储改写后的摘要。 + RetargetedRowDigest string `json:"retargeted_row_digest"` + // TargetRowDigest 是目标现状按归档列投影后的摘要:必须与上一条一致。 + TargetRowDigest string `json:"target_row_digest"` + // ProvenanceJobID / ProvenanceSnapshotID 指向一次**已成功**的同步作业及 + // 它当时写入目标的快照;ProvenanceRowDigest 是那份快照里同一行的摘要。 + ProvenanceJobID string `json:"provenance_job_id"` + ProvenanceSnapshotID string `json:"provenance_snapshot_id"` + ProvenanceKind string `json:"provenance_kind"` + ProvenanceRowDigest string `json:"provenance_row_digest"` + Objects []reuseObject `json:"objects,omitempty"` + CreatedAt time.Time `json:"created_at"` + Digest string `json:"digest"` +} + +func reusePath(archiveRoot, digest string) string { + return filepath.Join(archiveRoot, reuseDirName, digest+".json") +} + +// Seal 规范化并写入摘要;字段顺序由 json.Marshal 的结构体顺序固定。 +func (e *ResourceReuseEvidence) Seal() error { + copied := *e + copied.Digest = "" + raw, err := json.Marshal(copied) + if err != nil { + return fmt.Errorf("编码复用证据失败: %w", err) + } + sum := sha256.Sum256(raw) + e.Digest = hex.EncodeToString(sum[:]) + return nil +} + +// Verify 校验版本、必备身份与摘要;缺任何一项都不许拿它去放行覆盖。 +func (e *ResourceReuseEvidence) Verify() error { + if e.Version != reuseEvidenceVersion { + return fmt.Errorf("复用证据版本 %d 不受支持(当前 %d)", e.Version, reuseEvidenceVersion) + } + if e.JobID == "" || e.Target == "" || e.TargetSnapshotID == "" || e.ResourceID == "" || + e.ProvenanceJobID == "" || e.ProvenanceSnapshotID == "" || e.ProvenanceKind == "" { + return errors.New("复用证据缺少作业/目标/快照/来源身份") + } + if e.TenantID == 0 || e.KnowledgeBaseID == "" || e.SourceTenantID == 0 || e.SourceKnowledgeBaseID == "" { + return errors.New("复用证据缺少范围身份") + } + // 三份行摘要必须一致:来源行、映射后的行、目标现状是同一份内容。 + if e.RetargetedRowDigest == "" || e.TargetRowDigest != e.RetargetedRowDigest { + return errors.New("复用证据里目标行与本次要写的行不一致") + } + if e.ProvenanceRowDigest != e.ArchivedRowDigest { + return errors.New("复用证据里的来源行与本次快照行不一致") + } + copied := *e + copied.Digest = "" + raw, err := json.Marshal(copied) + if err != nil { + return err + } + sum := sha256.Sum256(raw) + if hex.EncodeToString(sum[:]) != e.Digest { + return errors.New("复用证据摘要不符(内容被改动或截断)") + } + return nil +} + +// PersistReuseEvidence 按内容寻址落盘(只读、幂等)。同摘要已存在时校验后直接返回。 +func PersistReuseEvidence(archiveRoot string, evidence *ResourceReuseEvidence) (string, error) { + if evidence.Digest == "" { + if err := evidence.Seal(); err != nil { + return "", err + } + } + if evidence.Digest == "" { + return "", errors.New("复用证据没有摘要,拒绝落盘") + } + directory := filepath.Join(archiveRoot, reuseDirName) + if err := os.MkdirAll(directory, 0700); err != nil { + return "", fmt.Errorf("创建复用证据目录失败: %w", err) + } + path := reusePath(archiveRoot, evidence.Digest) + if existing, readErr := os.ReadFile(path); readErr == nil { + var recorded ResourceReuseEvidence + if json.Unmarshal(existing, &recorded) != nil || recorded.Verify() != nil || recorded.Digest != evidence.Digest { + return "", errors.New("复用证据同名但内容不符,拒绝覆盖") + } + return path, nil + } else if !errors.Is(readErr, os.ErrNotExist) { + return "", fmt.Errorf("读取已有复用证据失败: %w", readErr) + } + raw, err := json.Marshal(*evidence) + if err != nil { + return "", fmt.Errorf("编码复用证据失败: %w", err) + } + temporary := path + ".tmp" + file, err := os.OpenFile(temporary, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return "", fmt.Errorf("写入复用证据失败: %w", err) + } + if _, err := file.Write(raw); err != nil { + file.Close() + os.Remove(temporary) + return "", fmt.Errorf("写入复用证据失败: %w", err) + } + if err := file.Sync(); err != nil { + file.Close() + os.Remove(temporary) + return "", fmt.Errorf("复用证据未落盘: %w", err) + } + if err := file.Close(); err != nil { + os.Remove(temporary) + return "", fmt.Errorf("复用证据未落盘: %w", err) + } + if err := os.Chmod(temporary, 0444); err != nil { + os.Remove(temporary) + return "", fmt.Errorf("复用证据只读化失败: %w", err) + } + if err := os.Rename(temporary, path); err != nil { + os.Remove(temporary) + return "", fmt.Errorf("复用证据落盘失败: %w", err) + } + return path, nil +} + +// LoadReuseEvidence 取回并校验证据;摘要或绑定对不上就失败。 +func LoadReuseEvidence(archiveRoot, digest string) (*ResourceReuseEvidence, error) { + if digest == "" { + return nil, errors.New("没有复用证据摘要") + } + raw, err := os.ReadFile(reusePath(archiveRoot, digest)) + if err != nil { + return nil, fmt.Errorf("复用证据缺失(%s)", digest) + } + var evidence ResourceReuseEvidence + if err := json.Unmarshal(raw, &evidence); err != nil { + return nil, errors.New("复用证据不可解析") + } + if err := evidence.Verify(); err != nil { + return nil, err + } + if evidence.Digest != digest { + return nil, errors.New("复用证据摘要不符") + } + return &evidence, nil +} + +// archivedRowByID 从归档部件里读出指定身份的行(原样,不改写)。 +func archivedRowByID(archive *librarysnapshot.Archive, part librarysnapshot.Part, id string) (map[string]json.RawMessage, error) { + reader, err := archive.OpenPart(part) + if err != nil { + return nil, fmt.Errorf("读取归档 %s 失败", part.Name) + } + defer reader.Close() + decoder := json.NewDecoder(reader) + token, err := decoder.Token() + if err != nil || token != json.Delim('[') { + return nil, fmt.Errorf("归档 %s 不是行数组", part.Name) + } + for decoder.More() { + var row map[string]json.RawMessage + if err := decoder.Decode(&row); err != nil { + return nil, fmt.Errorf("解析归档 %s 行失败", part.Name) + } + var identity string + if json.Unmarshal(row["id"], &identity) == nil && identity == id { + return row, nil + } + } + return nil, nil +} + +// targetResourceRow 读出目标里这一行;不存在返回 nil。 +func (t *ProductionLibraryTarget) targetResourceRow(ctx context.Context, id string) (map[string]json.RawMessage, error) { + var raw map[string]json.RawMessage + err := t.pool.QueryRow(ctx, `SELECT to_jsonb(r) FROM resources r WHERE r.id=$1 AND r.tenant_id=$2`, + id, t.scope.TenantID).Scan(&raw) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("读取目标资源 %s 失败", id) + } + return raw, nil +} + +// cloneRow 复制一行,避免改写归档缓存/调用方数据。 +func cloneRow(row map[string]json.RawMessage) map[string]json.RawMessage { + copied := make(map[string]json.RawMessage, len(row)) + for key, value := range row { + copied[key] = value + } + return copied +} + +// projectRow 只保留给定列集合里的列:目标行比归档多出的默认列不参与比对。 +func projectRow(row map[string]json.RawMessage, columns map[string]bool) map[string]json.RawMessage { + projected := make(map[string]json.RawMessage, len(columns)) + for key := range columns { + if value, ok := row[key]; ok { + projected[key] = value + } + } + return projected +} + +// targetObjectSHA256 读取目标对象的实际字节并返回大小与摘要。 +func (t *ProductionLibraryTarget) targetObjectSHA256(ctx context.Context, provider, path string) (int64, string, error) { + if t.config.ObjectsReader == nil { + return 0, "", errors.New("缺少目标对象读取器,无法证明对象字节一致") + } + reader, err := t.config.ObjectsReader.Open(ctx, provider, path) + if err != nil { + return 0, "", err + } + defer reader.Close() + hash := sha256.New() + size, err := io.Copy(hash, reader) + if err != nil { + return 0, "", err + } + return size, hex.EncodeToString(hash.Sum(nil)), nil +} + +// resourceLocatorField 解析改写后的资源行里的定位符与提供方(与写入/校验同一口径)。 +func resourceLocatorField(row map[string]json.RawMessage) (provider, path string, err error) { + if err := json.Unmarshal(row["provider"], &provider); err != nil { + return "", "", errors.New("资源行提供方不可解析") + } + if err := json.Unmarshal(row["physical_path"], &path); err != nil || path == "" { + return "", "", errors.New("资源行定位符不可解析") + } + return provider, path, nil +} + +// provenanceFor 在"这个目标的持久来源"里找同一资源的同一行内容: +// - 成功同步过的快照(说明这个目标先前就把这行写进去过); +// - 这个目标自己的预备份(说明这行本来就是本目标历史状态的一部分)。 +// +// 找不到就说明归属无法证明:即使目标里那行看起来一模一样,也不能覆盖/复用。 +func provenanceFor(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, targetSnapshotID, + id, archivedDigest string, prior []libraryrelease.PriorSnapshot) (libraryrelease.PriorSnapshot, string, error) { + for _, candidate := range prior { + if candidate.JobID == "" || candidate.SnapshotID == "" { + continue + } + if candidate.SnapshotID == targetSnapshotID { + // 本次要写的快照本身就是一次成功同步写进目标的那一份。 + return candidate, archivedDigest, nil + } + priorManifest, err := archive.Load(candidate.SnapshotID) + if err != nil { + // 来源快照不可读不能当作"没有来源":那会把不可审计变成静默放行。 + return libraryrelease.PriorSnapshot{}, "", fmt.Errorf("复用来源快照 %s 不可读", candidate.SnapshotID) + } + if priorManifest.TenantID != manifest.TenantID || priorManifest.KnowledgeBaseID != manifest.KnowledgeBaseID { + continue + } + var part librarysnapshot.Part + found := false + for _, item := range priorManifest.Parts { + if item.Kind == "database" && item.Name == resourceReuseTableName { + part, found = item, true + break + } + } + if !found { + continue + } + row, err := archivedRowByID(archive, part, id) + if err != nil { + return libraryrelease.PriorSnapshot{}, "", err + } + if row == nil { + continue + } + digest, err := canonicalRowDigest(row) + if err != nil { + return libraryrelease.PriorSnapshot{}, "", err + } + if digest != archivedDigest { + continue + } + return candidate, digest, nil + } + return libraryrelease.PriorSnapshot{}, "", nil +} + +// planReusableResource 是资源冲突的唯一放行路径:内容相同 + 字节相同 + 来源可审计。 +// 返回写集合行(Existing + Reused)并把它登记到本次目标,写入阶段据此跳过这一行。 +func (t *ProductionLibraryTarget) planReusableResource(ctx context.Context, archive *librarysnapshot.Archive, + manifest librarysnapshot.Manifest, part librarysnapshot.Part, id, owner string, + plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSetRow, error) { + archived, err := archivedRowByID(archive, part, id) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + if archived == nil { + return libraryrelease.WriteSetRow{}, fmt.Errorf("资源 %s 在快照 %s 的 resources 部件里找不到", id, plan.TargetSnapshotID) + } + archivedDigest, err := canonicalRowDigest(archived) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + columns := map[string]bool{} + for key := range archived { + columns[key] = true + } + retargeted := cloneRow(archived) + if err := t.retargetRow(resourceReuseTableName, retargeted); err != nil { + return libraryrelease.WriteSetRow{}, err + } + retargetedDigest, err := canonicalRowDigest(retargeted) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + existing, err := t.targetResourceRow(ctx, id) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + if existing == nil { + return libraryrelease.WriteSetRow{}, fmt.Errorf("资源 %s 在目标里已不存在,不能复用", id) + } + targetDigest, err := canonicalRowDigest(projectRow(existing, columns)) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + if targetDigest != retargetedDigest { + return libraryrelease.WriteSetRow{}, fmt.Errorf( + "资源 ID 冲突:%s 已存在于目标租户但内容与本次快照不一致(目标 %s / 本次 %s),拒绝覆盖", + id, shortDigest(targetDigest), shortDigest(retargetedDigest)) + } + provenance, provenanceDigest, err := provenanceFor(archive, manifest, plan.TargetSnapshotID, id, archivedDigest, plan.PriorSnapshots) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + if provenance.JobID == "" { + return libraryrelease.WriteSetRow{}, fmt.Errorf( + "资源 ID 冲突:%s 已存在于目标租户但不在预备份范围内,且没有可审计的持久来源(成功同步/预备份)证明它属于本目标,拒绝覆盖", id) + } + provider, path, err := resourceLocatorField(retargeted) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + objects := make([]reuseObject, 0, 2) + for _, item := range manifest.Parts { + if item.Kind != "resource" || item.Name != id { + continue + } + size, digest, err := t.targetObjectSHA256(ctx, provider, path) + if err != nil { + return libraryrelease.WriteSetRow{}, fmt.Errorf("资源 %s 的对象读不到,无法证明字节一致:%w", id, err) + } + if size != item.Bytes || digest != item.SHA256 { + return libraryrelease.WriteSetRow{}, fmt.Errorf( + "资源 ID 冲突:%s 的目标对象字节与本次快照不一致(%d/%d 字节),拒绝覆盖", + id, size, item.Bytes) + } + objects = append(objects, reuseObject{Name: item.Name, Path: path, Bytes: size, SHA256: digest}) + } + evidence := &ResourceReuseEvidence{ + Version: reuseEvidenceVersion, JobID: plan.JobID, Target: plan.Target, + TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID, + SourceTenantID: manifest.TenantID, SourceKnowledgeBaseID: manifest.KnowledgeBaseID, + TargetSnapshotID: plan.TargetSnapshotID, ResourceID: id, + ArchivedRowDigest: archivedDigest, RetargetedRowDigest: retargetedDigest, TargetRowDigest: targetDigest, + ProvenanceJobID: provenance.JobID, ProvenanceSnapshotID: provenance.SnapshotID, ProvenanceKind: provenance.Kind, + ProvenanceRowDigest: provenanceDigest, Objects: objects, CreatedAt: t.now().UTC(), + } + if _, err := PersistReuseEvidence(t.config.ArchiveRoot, evidence); err != nil { + return libraryrelease.WriteSetRow{}, fmt.Errorf("复用证据无法落盘,拒绝放行覆盖:%w", err) + } + if t.reusedResources == nil { + t.reusedResources = map[string]bool{} + } + t.reusedResources[id] = true + return libraryrelease.WriteSetRow{Table: resourceReuseTableName, ID: id, Existing: true, + Owner: owner, Reused: true, ReuseEvidence: evidence.Digest}, nil +} + +func shortDigest(digest string) string { + if len(digest) > 12 { + return digest[:12] + } + return digest +} diff --git a/internal/releaseadapter/v071/library_reuse_integration_test.go b/internal/releaseadapter/v071/library_reuse_integration_test.go new file mode 100644 index 0000000..7f81cf3 --- /dev/null +++ b/internal/releaseadapter/v071/library_reuse_integration_test.go @@ -0,0 +1,305 @@ +package v071 + +import ( + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/libraryrelease" +) + +// 这一组用例覆盖真实缺陷现场:同步 A 成功 → 手工恢复 B(对象保留、资源行不删) +// → 再同步 A。旧预检把"恢复保留下来的、本目标自己上次写进去的资源"判成 +// "其它知识库的同 ID 冲突",整个同步在写第一笔之前被拒。 +// +// 复用的放行条件必须是可审计的证据,而不是"目标里没人引用": +// - 目标现状那一行与本次要写的行逐列相同(除易变时间戳); +// - 该资源的对象字节与本次快照归档摘要一致; +// - 治理库里存在"这个目标成功同步过、且含有同一行内容"的快照/作业。 +// 缺任何一条都必须继续拒绝,且拒绝发生在写入之前。 + +const ( + wsHandleNew = "gggggggggggggggggggggg" + // resNewBody 是"A 的下一次发布"新增资源的内容;与 res-src 的 "12345678" 不同, + // 这样"本次真新建"和"证明可复用"在同一份写集合里能同时被观察到。 + resNewBody = "87654321" +) + +// reusePriors 是治理库会给出的来源清单:一次成功同步 + 它产生的预备份。 +func reusePriors(syncJobID, syncSnapshotID string, prebackups ...string) []libraryrelease.PriorSnapshot { + priors := []libraryrelease.PriorSnapshot{{JobID: syncJobID, SnapshotID: syncSnapshotID, Kind: libraryrelease.PriorSnapshotSync}} + for _, snapshot := range prebackups { + priors = append(priors, libraryrelease.PriorSnapshot{JobID: syncJobID, SnapshotID: snapshot, + Kind: libraryrelease.PriorSnapshotPrebackup}) + } + return priors +} + +func requestForWithPrior(snapshotID, jobID string, prior []libraryrelease.PriorSnapshot) libraryrelease.Request { + request := requestFor(snapshotID) + request.JobID = jobID + request.PriorSnapshots = prior + return request +} + +// syncThenRestore 造出现场:同步 A 成功,然后按生产恢复作业的方式恢复到 A 的 +// 预备份(再走一次 Run:新的预备份 + 它自己的写集合)。恢复不删 resources 表, +// 所以 A 写进去的资源会作为"没人引用的行"留在目标里——正是本次缺陷的前提。 +func syncThenRestore(t *testing.T, fx *writeSetFixture, snapshotID string) (*ProductionLibraryTarget, + *ProductionBackupStore, libraryrelease.Outcome, []libraryrelease.PriorSnapshot) { + t.Helper() + target := fx.target(t) + backups := fx.backups(t, target) + first := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a", nil), target, backups, nil) + if first.State != "succeeded" { + t.Fatalf("首次同步 A 必须成功:%+v", first) + } + if first.Backup.WriteSet == nil { + t.Fatal("首次同步必须落盘写集合") + } + // 恢复作业同样只认已登记来源:它自己刚拍的那份预备份就是本目标的历史状态。 + restore := libraryrelease.Run(fx.ctx, requestForWithPrior(first.Backup.SnapshotID, "job-restore-b", + reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)), target, backups, nil) + if restore.State != "succeeded" { + t.Fatalf("手工恢复 B 必须成功:%+v", restore) + } + var bindings int64 + if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resource_bindings WHERE resource_id='res-src'`).Scan(&bindings); err != nil { + t.Fatal(err) + } + if bindings != 0 { + t.Fatalf("恢复后 res-src 不应还有绑定行(现场前提):%d", bindings) + } + return target, backups, first, reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID) +} + +// resourceFingerprint 取 PG 行的 xmin 与整行 JSON:xmin 不变才能证明"没有重写这一行"。 +func resourceFingerprint(t *testing.T, pool *pgxpool.Pool, id string) (string, string) { + t.Helper() + var xmin, row string + if err := pool.QueryRow(t.Context(), `SELECT xmin::text, to_jsonb(r)::text FROM resources r WHERE id=$1`, id). + Scan(&xmin, &row); err != nil { + t.Fatalf("读取资源 %s: %v", id, err) + } + return xmin, row +} + +func resourceRowOf(t *testing.T, pool *pgxpool.Pool, id string) string { + t.Helper() + var row string + if err := pool.QueryRow(t.Context(), `SELECT to_jsonb(r)::text FROM resources r WHERE id=$1`, id).Scan(&row); err != nil { + t.Fatalf("读取资源 %s: %v", id, err) + } + return row +} + +func TestLibraryResyncAfterRestoreReusesProvenResourcesIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + snapshotID := sealSourceSnapshot(t, fx, nil) + target, backups, first, priors := syncThenRestore(t, fx, snapshotID) + + beforeXmin, beforeRow := resourceFingerprint(t, fx.pool, "res-src") + orphanBefore := resourceRowOf(t, fx.pool, "res-orphan") + otherBefore := resourceRowOf(t, fx.pool, "res-other") + sharedBefore := resourceRowOf(t, fx.pool, "res-shared") + foreignBefore := resourceRowOf(t, fx.pool, "res-foreign") + + outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a-again", priors), + target, backups, nil) + if outcome.State != "succeeded" { + t.Fatalf("恢复后再次同步 A 必须完成:%+v", outcome) + } + if outcome.Backup.WriteSet == nil { + t.Fatal("写集合必须落盘") + } + ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest) + if err != nil { + t.Fatalf("写集合记录取不回来:%v", err) + } + reused := ledger.ReusedIDs("resources") + if len(reused) != 1 || reused[0] != "res-src" { + t.Fatalf("复用资源 = %v,want [res-src]", reused) + } + if created := ledger.CreatedIDs("resources"); len(created) != 0 { + t.Fatalf("复用资源不能被记成本次新建(恢复会删掉它):%v", created) + } + var reuseRow libraryrelease.WriteSetRow + for _, row := range ledger.Rows { + if row.Table == "resources" && row.ID == "res-src" { + reuseRow = row + } + } + if !reuseRow.Existing || !reuseRow.Reused || reuseRow.ReuseEvidence == "" { + t.Fatalf("复用行必须带证据摘要并标记为写前已存在:%+v", reuseRow) + } + evidence, err := LoadReuseEvidence(fx.archiveRoot, reuseRow.ReuseEvidence) + if err != nil { + t.Fatalf("复用证据取不回来:%v", err) + } + if evidence.ProvenanceSnapshotID != snapshotID || evidence.ProvenanceJobID != first.Backup.WriteSet.JobID { + t.Fatalf("复用证据没有指向前一次成功同步:%+v", evidence) + } + if evidence.TargetRowDigest != evidence.RetargetedRowDigest || evidence.ProvenanceRowDigest != evidence.ArchivedRowDigest { + t.Fatalf("复用证据的行摘要不一致:%+v", evidence) + } + if len(evidence.Objects) != 1 || evidence.Objects[0].SHA256 == "" { + t.Fatalf("复用证据必须逐个记录对象字节:%+v", evidence.Objects) + } + + // 复用不许改原行:xmin 与整行 JSON 都必须原样。 + afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src") + if afterXmin != beforeXmin { + t.Fatalf("复用资源被重写了:xmin %s -> %s", beforeXmin, afterXmin) + } + if afterRow != beforeRow { + t.Fatalf("复用资源内容变了:\n%s\n->\n%s", beforeRow, afterRow) + } + // 历史孤儿/其它知识库/共享/外租户资源一律不许被覆盖。 + for id, want := range map[string]string{"res-orphan": orphanBefore, "res-other": otherBefore, + "res-shared": sharedBefore, "res-foreign": foreignBefore} { + if got := resourceRowOf(t, fx.pool, id); got != want { + t.Fatalf("资源 %s 被改动:\n%s\n->\n%s", id, want, got) + } + } +} + +func TestLibraryResyncAfterRestoreRejectsReuseWithoutProvenanceIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + snapshotID := sealSourceSnapshot(t, fx, nil) + target, backups, first, _ := syncThenRestore(t, fx, snapshotID) + before := universeDump(t, fx.pool) + + // 已登记的来源里没有 res-src(只有这次同步之前的预备份):即使目标那一行 + // 看起来一模一样,也不许放行。 + noSource := []libraryrelease.PriorSnapshot{{JobID: first.Backup.WriteSet.JobID, + SnapshotID: first.Backup.SnapshotID, Kind: libraryrelease.PriorSnapshotPrebackup}} + outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-no-provenance", noSource), target, backups, nil) + if outcome.State != "failed" { + t.Fatalf("没有来源证据时必须拒绝:%+v", outcome) + } + if outcome.FailedAt != "" { + t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at=%s", outcome.FailedAt) + } + if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "可审计") { + t.Fatalf("拒绝原因必须点名资源并说明缺来源证据:%s", outcome.Error) + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("被拒绝的同步不许动目标:%s", diff) + } +} + +func TestLibraryResyncAfterRestoreRejectsReuseWhenContentDiffersIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + snapshotID := sealSourceSnapshot(t, fx, nil) + target, backups, _, priors := syncThenRestore(t, fx, snapshotID) + // 同 ID 同租户,但内容被改过(这里改大小,等价于"另一份文件占了这个 ID")。 + if _, err := fx.pool.Exec(fx.ctx, `UPDATE resources SET size=size+1 WHERE id='res-src'`); err != nil { + t.Fatal(err) + } + changed := resourceRowOf(t, fx.pool, "res-src") + before := universeDump(t, fx.pool) + outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-content-diff", priors), target, backups, nil) + if outcome.State != "failed" { + t.Fatalf("内容不一致必须拒绝:%+v", outcome) + } + if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "内容") { + t.Fatalf("拒绝原因必须点名内容不一致:%s", outcome.Error) + } + if resourceRowOf(t, fx.pool, "res-src") != changed { + t.Fatal("被拒绝的同步不许改目标资源") + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("被拒绝的同步不许动目标:%s", diff) + } +} + +// 同租户、同 ID、同内容,但来源快照属于另一个知识库:仍然不许复用。 +// "归属无法证明"包括"来源不同",不能因为字节一样就把别人的行当成自己的。 +func TestLibraryReuseRequiresSameSourceScopeIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + snapshotID := sealSourceSnapshot(t, fx, nil) + target, backups, _, priors := syncThenRestore(t, fx, snapshotID) + otherSource := sealSourceSnapshotScope(t, fx, wsSourceTenant+1, "kb-source-other", nil, nil) + before := universeDump(t, fx.pool) + outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(otherSource, "job-sync-other-source", priors), target, backups, nil) + if outcome.State != "failed" { + t.Fatalf("来源范围不同必须拒绝复用:%+v", outcome) + } + if !strings.Contains(outcome.Error, "可审计") { + t.Fatalf("拒绝原因必须说明来源不可审计:%s", outcome.Error) + } + if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" { + t.Fatalf("被拒绝的同步不许动目标:%s", diff) + } +} + +// 部分写失败:database/resources 已提交、wiki 因重复 slug 失败 → 回滚只清"本次真新建", +// 复用资源与历史孤儿原内容完整保留(xmin 也不变)。 +func TestLibraryHalfWriteRollbackKeepsReusedResourcesIsolated(t *testing.T) { + fx := openWriteSetFixture(t) + seedTargetData(t, fx) + snapshotID := sealSourceSnapshot(t, fx, nil) + target, backups, _, priors := syncThenRestore(t, fx, snapshotID) + reuseXmin, reuseRow := resourceFingerprint(t, fx.pool, "res-src") + orphanBefore := resourceRowOf(t, fx.pool, "res-orphan") + + next := sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB, + func(parts map[string][]map[string]any) { + parts["chunks"] = append(parts["chunks"], map[string]any{"id": "chunk-src-2", "tenant_id": wsSourceTenant, + "knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src", "content": "新增正文 resource://" + wsHandleNew}) + parts["resources"] = append(parts["resources"], map[string]any{"id": "res-new", "handle": wsHandleNew, + "tenant_id": wsSourceTenant, "storage_backend_id": "backend-source", "provider": "oss", + "physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-new"), + "size": int64(len(resNewBody))}) + parts["resource_bindings"] = append(parts["resource_bindings"], map[string]any{"id": "bind-src-2", + "resource_id": "res-new", "tenant_id": wsSourceTenant, "owner_type": "chunk", + "owner_id": "chunk-src-2", "relation": "attachment"}) + // 同一作用域两页同 slug:wiki 的唯一索引让 wiki 阶段失败(真实半写现场)。 + parts["wiki_pages"] = []map[string]any{ + {"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"}, + {"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"}, + } + }, map[string]string{"res-new": resNewBody}) + + outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(next, "job-sync-half-write", priors), target, backups, nil) + if outcome.State != "rolled_back" { + t.Fatalf("wiki 半写必须回滚:%+v", outcome) + } + if outcome.FailedAt != libraryrelease.StageWiki { + t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt) + } + ledger := outcome.Backup.WriteSet + if ledger == nil { + t.Fatal("半写作业必须留下写集合") + } + if created := ledger.CreatedIDs("resources"); len(created) != 1 || created[0] != "res-new" { + t.Fatalf("本次真新建的资源 = %v,want [res-new]", created) + } + if reused := ledger.ReusedIDs("resources"); len(reused) != 1 || reused[0] != "res-src" { + t.Fatalf("复用资源 = %v,want [res-src]", reused) + } + // 本次真新建的清干净。 + var residue int64 + if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-new'`).Scan(&residue); err != nil { + t.Fatal(err) + } + if residue != 0 { + t.Fatal("回滚后本次新建的资源必须被清理") + } + // 复用资源与历史孤儿原内容(含 xmin)保留。 + afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src") + if afterXmin != reuseXmin || afterRow != reuseRow { + t.Fatalf("回滚动了复用资源:xmin %s->%s\n%s\n->\n%s", reuseXmin, afterXmin, reuseRow, afterRow) + } + if resourceRowOf(t, fx.pool, "res-orphan") != orphanBefore { + t.Fatal("回滚动了历史孤儿资源") + } + if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil { + t.Fatalf("回滚必须与预备份一致:%v", err) + } +} diff --git a/internal/releaseadapter/v071/library_writeset.go b/internal/releaseadapter/v071/library_writeset.go index faeaa0e..cee567e 100644 --- a/internal/releaseadapter/v071/library_writeset.go +++ b/internal/releaseadapter/v071/library_writeset.go @@ -132,6 +132,14 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library if len(plan.Universe) == 0 { return libraryrelease.WriteSet{}, errors.New("预备份没有整实例内容证据,拒绝在没有保护证据的情况下写目标") } + // 资源复用的"整行内容"必须按写入时的同一套范围/存储改写来比,所以先把目标 + // 存储后端解析出来;解析不到就等于写不出可解析的定位符,直接拒绝。 + if err := t.resolveStorageBackend(ctx); err != nil { + return libraryrelease.WriteSet{}, err + } + // 每次预检都从空集开始:同一进程里复用目标对象时,上一次作业的"可复用" + // 判断不能泄漏到这一次。 + t.reusedResources = map[string]bool{} archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) if err != nil { return libraryrelease.WriteSet{}, errors.New("归档不可用") @@ -173,6 +181,20 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id}) continue } + if part.Name == resourceReuseTableName && !backupResources[id] { + // 资源不在预备份范围,但可能正是这个目标上一次成功同步写进去、 + // 又被恢复保留下来的那份内容。只有"内容/字节相同 + 来源可审计" + // 才允许复用(不重写、不清理);否则按跨范围冲突拒绝。 + if err := t.assertResourceTenant(id, owner); err != nil { + return libraryrelease.WriteSet{}, err + } + row, reuseErr := t.planReusableResource(ctx, archive, manifest, part, id, owner, plan) + if reuseErr != nil { + return libraryrelease.WriteSet{}, reuseErr + } + set.Rows = append(set.Rows, row) + continue + } if err := t.assertWriteSetOwnership(part.Name, id, owner, backupResources); err != nil { return libraryrelease.WriteSet{}, err } @@ -265,6 +287,15 @@ WHERE b.id = ANY($1::text[])` return owners, rows.Err() } +// assertResourceTenant 只检查"同 ID 是否属于别的租户":resources 没有 +// knowledge_base_id,同租户不等于同知识库,范围问题交给后续的复用证明。 +func (t *ProductionLibraryTarget) assertResourceTenant(id, owner string) error { + if owner != fmt.Sprintf("%d/", t.scope.TenantID) { + return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner) + } + return nil +} + // assertWriteSetOwnership 是跨范围 ID 冲突的预检:同 ID 已经是别人的行时, // 写入会改掉别人的数据(resources 尤其危险:ON CONFLICT 会整行覆盖)。 // 宁可整次拒绝,也不许"覆盖一下看起来也能跑"。 @@ -272,8 +303,8 @@ func (t *ProductionLibraryTarget) assertWriteSetOwnership(table, id, owner strin target := fmt.Sprintf("%d/%s", t.scope.TenantID, t.scope.KnowledgeBaseID) if table == "resources" { // 资源只有租户:同租户不等于同知识库,必须落在预备份范围内才算自己的。 - if owner != fmt.Sprintf("%d/", t.scope.TenantID) { - return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner) + if err := t.assertResourceTenant(id, owner); err != nil { + return err } if !backupResources[id] { return fmt.Errorf("资源 ID 冲突:%s 存在于目标租户但不在预备份范围内(属于其它知识库),拒绝覆盖", id) diff --git a/internal/releaseadapter/v071/library_writeset_integration_test.go b/internal/releaseadapter/v071/library_writeset_integration_test.go index 3bb293b..15ef1e2 100644 --- a/internal/releaseadapter/v071/library_writeset_integration_test.go +++ b/internal/releaseadapter/v071/library_writeset_integration_test.go @@ -293,6 +293,14 @@ func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb, // sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。 func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string { + t.Helper() + return sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB, tweak, nil) +} + +// sealSourceSnapshotScope 允许指定来源范围/附加对象:来源快照的身份决定 +// "同一资源行是否属于同一个来源",复用证据必须按它区分。 +func sealSourceSnapshotScope(t *testing.T, fx *writeSetFixture, sourceTenant uint64, sourceKB string, + tweak func(parts map[string][]map[string]any), extraObjects map[string]string) string { t.Helper() archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024) if err != nil { @@ -307,20 +315,20 @@ func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[ return encoded } parts := map[string][]map[string]any{ - "knowledges": {{"id": "doc-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, + "knowledges": {{"id": "doc-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}}, - "chunks": {{"id": "chunk-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, + "chunks": {{"id": "chunk-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}}, "knowledge_tags": {}, "knowledge_tag_relations": {}, - "resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": wsSourceTenant, + "resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": sourceTenant, "storage_backend_id": "backend-source", "provider": "oss", - "physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-src"), + "physical_path": wsLocator("backend-source", "oss", "source-bucket", sourceTenant, "res-src"), "size": int64(8)}}, - "resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": wsSourceTenant, + "resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": sourceTenant, "owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}}, "wiki_folders": {}, - "wiki_pages": {{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"}}, + "wiki_pages": {{"id": "wiki-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "slug": "index", "content": "来源 wiki"}}, "wiki_page_issues": {}, "wiki_log_entries": {}, } @@ -350,17 +358,38 @@ func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[ } resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1 manifestParts = append(manifestParts, resourcePart) - indexPart, err := archive.Put(bytes.NewReader(encode([]map[string]any{{"id": "chunk-src", "source": map[string]any{ - "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src", - "chunk_id": "chunk-src", "source_id": "chunk-src", "content": "来源正文 resource://" + wsHandleSource, - "embedding": []float64{0.1, 0.2}}}}))) + extraNames := make([]string, 0, len(extraObjects)) + for name := range extraObjects { + extraNames = append(extraNames, name) + } + sort.Strings(extraNames) + for _, name := range extraNames { + part, err := archive.Put(bytes.NewReader([]byte(extraObjects[name]))) + if err != nil { + t.Fatal(err) + } + part.Kind, part.Name, part.Count = "resource", name, 1 + manifestParts = append(manifestParts, part) + } + // 索引文档按快照里的分块逐条生成:多一个分块就必须多一条向量,否则 + // 校验会把"写入正确"判成索引缺文档。 + indexRows := make([]map[string]any, 0, len(parts["chunks"])) + for _, chunk := range parts["chunks"] { + id, _ := chunk["id"].(string) + knowledge, _ := chunk["knowledge_id"].(string) + content, _ := chunk["content"].(string) + indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{ + "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "knowledge_id": knowledge, + "chunk_id": id, "source_id": id, "content": content, "embedding": []float64{0.1, 0.2}}}) + } + indexPart, err := archive.Put(bytes.NewReader(encode(indexRows))) if err != nil { t.Fatal(err) } - indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", 1 + indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows)) manifestParts = append(manifestParts, indexPart) manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy, - Format: librarysnapshot.Format, Scope: "internal", TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB, + Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB, CreatedAt: time.Now().UTC(), Parts: manifestParts, Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",