fix(library): 恢复保留下来的资源凭可审计证据复用
同步 A 成功后手工恢复 B,resources 表既不删行也不删对象;再同步 A 时, resources 里那 21 行以“同租户但不在预备份范围”被判成别人的知识库,整次 同步在写第一笔之前被拒。 “目标里没人引用”不是归属证据:历史孤儿、其它知识库暂未绑定的资源都满足 它。改成唯一放行路径——可审计复用(v071/library_reuse.go): 1. 持久来源:这个目标成功同步过且登记过生效版本的快照,或它自己的预备份, 且同一资源行在来源里内容一致; 2. 整行内容(除易变时间戳)与本次写入将要落地的行完全一致; 3. 该资源的对象字节与本次快照归档摘要一致。 三条同时成立才允许复用:写入阶段跳过它(不重写行、不重传对象),写集合里 记 existing+reused,恢复不清理;证据单独落盘 reuse/<digest>.json,摘要写进 写集合行,不冒充历史台账。内容不同、来源范围不同、或没有持久来源一律继续 拒绝,且拒绝发生在写入之前。 为什么把预备份也算来源:恢复作业的写集合同样会把“本目标历史状态里已有的 资源”判成跨范围冲突,而它们本来就出自这个目标自己的快照。 验证:隔离 PG+ES+MinIO 真服务下新增 5 个场景用例(复用成功且 xmin 不变、 无来源拒绝、内容不同拒绝、同内容不同来源拒绝、wiki 半写回滚只清本次新建); v071 整包与全仓 go test 通过;make verify 15 项全绿。
This commit is contained in:
@@ -69,7 +69,13 @@ func (t *contentTrackedTarget) PlanWriteSet(ctx context.Context, plan libraryrel
|
||||
if !ok {
|
||||
return libraryrelease.WriteSet{}, errors.New("目标实现不支持写集合预检:拒绝在没有可追溯写集合的情况下写生产")
|
||||
}
|
||||
return planner.PlanWriteSet(ctx, plan)
|
||||
set, err := planner.PlanWriteSet(ctx, plan)
|
||||
if err == nil {
|
||||
// 日志是"wrapper 路径真的执行了预检"的现场证据:接管排查先看这一行。
|
||||
log.Printf("library writeset planned job=%s target=%s digest=%s rows=%d reused=%d target_snapshot=%s",
|
||||
set.JobID, set.Target, set.Digest, len(set.Rows), len(set.ReusedIDs("resources")), set.TargetSnapshotID)
|
||||
}
|
||||
return set, err
|
||||
}
|
||||
|
||||
// libraryReleasePayload 是同步与恢复共用的队列载荷形状。
|
||||
@@ -157,6 +163,17 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
if err != nil || !found {
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "没有与目标快照匹配的有效发布授权", "prebackup")
|
||||
}
|
||||
// 资源复用的来源证据:只认这个目标"成功同步过且登记过生效版本"的快照与它自己
|
||||
// 的预备份。读不到就按空处理——那只会让复用被拒绝,绝不会误放行覆盖。
|
||||
priorSnapshots := make([]libraryrelease.PriorSnapshot, 0, 8)
|
||||
if sources, sourceErr := store.LibraryReuseSources(ctx, payload.Scope, 100); sourceErr != nil {
|
||||
log.Printf("library %s prior reuse sources unavailable: %v", payload.LibraryJobID, sourceErr)
|
||||
} else {
|
||||
for _, source := range sources {
|
||||
priorSnapshots = append(priorSnapshots, libraryrelease.PriorSnapshot{JobID: source.JobID,
|
||||
SnapshotID: source.SnapshotID, Kind: source.Kind, FinishedAt: source.FinishedAt})
|
||||
}
|
||||
}
|
||||
// 生产索引映射基线:优先当前生效版本,其次任何一次成功预备份(它也记录了生产实际映射)。
|
||||
// 只认"生效版本"会在恢复把版本 supersede 掉之后误判"索引被改动"。
|
||||
knownMapping := ""
|
||||
@@ -325,6 +342,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
|
||||
request := libraryrelease.Request{JobID: payload.LibraryJobID, Target: payload.Scope,
|
||||
SnapshotID: payload.TargetSnapshotID, ParamsDigest: digest, Now: now,
|
||||
PriorSnapshots: priorSnapshots,
|
||||
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
|
||||
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
|
||||
|
||||
@@ -84,6 +84,7 @@ func writeSetEvidence(existing map[string]any, set libraryrelease.WriteSet, mapp
|
||||
evidence["writeset_universe_digest"] = set.UniverseDigest()
|
||||
evidence["writeset_target_snapshot"] = set.TargetSnapshotID
|
||||
evidence["writeset_rows"] = len(set.Rows)
|
||||
evidence["writeset_reused_rows"] = len(set.ReusedIDs("resources"))
|
||||
return evidence
|
||||
}
|
||||
|
||||
|
||||
@@ -167,10 +167,12 @@ func (t *plainTarget) Restore(context.Context, libraryrelease.BackupRef) error
|
||||
type planningTarget struct {
|
||||
plainTarget
|
||||
planned int
|
||||
last libraryrelease.WriteSetPlan
|
||||
}
|
||||
|
||||
func (t *planningTarget) PlanWriteSet(context.Context, libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
|
||||
func (t *planningTarget) PlanWriteSet(_ context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
|
||||
t.planned++
|
||||
t.last = plan
|
||||
return libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: "job-1", Target: "internal"}, nil
|
||||
}
|
||||
|
||||
@@ -181,12 +183,17 @@ func TestContentTrackedTargetForwardsWriteSetPlanning(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatal("包装后的目标必须仍然实现 WriteSetPlanner,否则写集合会被静默跳过")
|
||||
}
|
||||
if _, err := planner.PlanWriteSet(context.Background(), libraryrelease.WriteSetPlan{JobID: "job-1", Target: "internal"}); err != nil {
|
||||
if _, err := planner.PlanWriteSet(context.Background(), libraryrelease.WriteSetPlan{JobID: "job-1", Target: "internal",
|
||||
PriorSnapshots: []libraryrelease.PriorSnapshot{{JobID: "job-0", SnapshotID: "snap-0", Kind: libraryrelease.PriorSnapshotSync}}}); err != nil {
|
||||
t.Fatalf("plan: %v", err)
|
||||
}
|
||||
if inner.planned != 1 {
|
||||
t.Fatalf("预检必须落到内层目标,planned=%d", inner.planned)
|
||||
}
|
||||
// 来源证据也必须透传:丢掉它就会让"恢复保留下来的资源"永远无法证明可复用。
|
||||
if len(inner.last.PriorSnapshots) != 1 || inner.last.PriorSnapshots[0].SnapshotID != "snap-0" {
|
||||
t.Fatalf("PriorSyncs 没有透传到内层:%+v", inner.last)
|
||||
}
|
||||
|
||||
// 内层不支持预检 ⇒ 硬错误,不许"没有台账也照写"。
|
||||
strict := &contentTrackedTarget{inner: &plainTarget{}, onApply: func() {}}
|
||||
|
||||
@@ -45,6 +45,10 @@ type Request struct {
|
||||
ParamsDigest string
|
||||
Grant Grant
|
||||
Now time.Time
|
||||
// PriorSnapshots 是这个目标的持久来源(成功同步过的快照 + 自己的预备份),
|
||||
// 由治理库提供。资源复用只认它作证据;为空时"目标里已有但不在预备份范围"
|
||||
// 的资源一律按冲突拒绝。
|
||||
PriorSnapshots []PriorSnapshot
|
||||
}
|
||||
|
||||
// CurrentState 是生产当前整库的指纹,用于预备份与最终验证。
|
||||
@@ -213,7 +217,8 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor
|
||||
if backup.WriteSet == nil {
|
||||
if planner, ok := target.(WriteSetPlanner); ok {
|
||||
set, planErr := planner.PlanWriteSet(ctx, WriteSetPlan{JobID: request.JobID, Target: request.Target,
|
||||
BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: request.SnapshotID, Universe: backup.Universe})
|
||||
BackupSnapshotID: backup.SnapshotID, TargetSnapshotID: request.SnapshotID, Universe: backup.Universe,
|
||||
PriorSnapshots: request.PriorSnapshots})
|
||||
if planErr != nil {
|
||||
outcome.State = "failed"
|
||||
outcome.Error = "写集合/冲突预检失败,未对生产执行任何写入:" + planErr.Error()
|
||||
|
||||
@@ -28,6 +28,16 @@ type WriteSetRow struct {
|
||||
Existing bool `json:"existing"`
|
||||
// Owner 是写前该行的归属("<tenant>/<knowledge_base>"),不存在时为空。
|
||||
Owner string `json:"owner,omitempty"`
|
||||
// Reused 表示这一行虽然写前就在目标里、却不在预备份范围内,但已经用
|
||||
// "持久作业/快照证据 + 整行内容 + 对象字节"证明与本次要写的内容完全一致,
|
||||
// 因此本次既不重写它、也不在恢复时清理它。
|
||||
//
|
||||
// 不能只因为"它没被任何绑定/正文引用"就当成自己的:历史孤儿、同租户另一个
|
||||
// 知识库暂未绑定的资源同样满足"没人引用",覆盖或删除同样是数据丢失。
|
||||
Reused bool `json:"reused,omitempty"`
|
||||
// ReuseEvidence 是可复用证据文档的摘要(reuse/<digest>.json),独立于写集合,
|
||||
// 防止把这次推断伪装成历史台账。
|
||||
ReuseEvidence string `json:"reuse_evidence,omitempty"`
|
||||
}
|
||||
|
||||
// WriteSetPlan 是调用方在写第一笔之前交给目标的信息:绑定作业、目标、预备份与快照,
|
||||
@@ -38,6 +48,29 @@ type WriteSetPlan struct {
|
||||
BackupSnapshotID string
|
||||
TargetSnapshotID string
|
||||
Universe map[string]TableFingerprint
|
||||
// PriorSnapshots 是治理库里"这个目标的持久来源":成功同步过的快照,以及这个
|
||||
// 目标自己的预备份。资源复用必须沿它把来源追到真实登记过的快照,而不是按
|
||||
// "目标里没人引用"推断归属。
|
||||
PriorSnapshots []PriorSnapshot
|
||||
}
|
||||
|
||||
// 复用来源的两种合法类别。
|
||||
const (
|
||||
// PriorSnapshotSync:这个目标成功同步过这份快照(且登记了生效版本)。
|
||||
PriorSnapshotSync = "sync"
|
||||
// PriorSnapshotPrebackup:这份快照是从这个目标自己身上拍的预备份,把它写回去
|
||||
// 就是回到本目标的历史状态。
|
||||
PriorSnapshotPrebackup = "prebackup"
|
||||
)
|
||||
|
||||
// PriorSnapshot 是一次**已登记**的来源:作业身份 + 快照身份 + 来源类别。
|
||||
// 只有它也含有同一个资源 ID、且行内容完全一致,才允许把目标里那行当作
|
||||
// "本次写入的等价物"。
|
||||
type PriorSnapshot struct {
|
||||
JobID string `json:"job_id"`
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Kind string `json:"kind"`
|
||||
FinishedAt time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
|
||||
// WriteSet 是"这一次发布的写集合 + 写前整实例内容证据",内容寻址、可校验、
|
||||
@@ -124,6 +157,14 @@ func (w *WriteSet) Verify() error {
|
||||
if w.Digest == "" {
|
||||
return errors.New("写集合记录没有摘要")
|
||||
}
|
||||
for _, row := range w.Rows {
|
||||
if row.Reused && !row.Existing {
|
||||
return fmt.Errorf("写集合记录把 %s/%s 标成可复用却不是写前已存在行", row.Table, row.ID)
|
||||
}
|
||||
if row.Reused && row.ReuseEvidence == "" {
|
||||
return fmt.Errorf("写集合记录把 %s/%s 标成可复用却没有证据摘要", row.Table, row.ID)
|
||||
}
|
||||
}
|
||||
digest, err := w.computeDigest()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -158,6 +199,19 @@ func (w WriteSet) ExistingIDs(table string) []string {
|
||||
return ids
|
||||
}
|
||||
|
||||
// ReusedIDs 返回本次"证明可复用、因此既不重写也不清理"的精确身份。
|
||||
// 它与 CreatedIDs 互斥,恢复清理一律不看这里(复用行必须原样保留)。
|
||||
func (w WriteSet) ReusedIDs(table string) []string {
|
||||
ids := make([]string, 0, 8)
|
||||
for _, row := range w.Rows {
|
||||
if row.Table == table && row.Reused {
|
||||
ids = append(ids, row.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
// UniverseDigest 是整实例证据的摘要,随台账一起登记,便于独立核对。
|
||||
func (w WriteSet) UniverseDigest() string {
|
||||
if len(w.Universe) == 0 {
|
||||
|
||||
@@ -102,3 +102,65 @@ func TestWriteSetCreatedIDsOnlyCoversRowsCreatedByThisRelease(t *testing.T) {
|
||||
t.Fatal("universe digest must be derived from the whole-instance evidence")
|
||||
}
|
||||
}
|
||||
|
||||
// 复用行是"写前已存在 + 有证据摘要"的一种:既不算本次新建(恢复不许删),
|
||||
// 也不能没有证据摘要,更不能被标成新建。
|
||||
func TestWriteSetReusedRowsStayOutOfCleanupAndRequireEvidence(t *testing.T) {
|
||||
set := WriteSet{Version: WriteSetVersion, JobID: "job-2", Target: "internal", TenantID: 10001,
|
||||
KnowledgeBaseID: "kb", BackupSnapshotID: "backup-2", TargetSnapshotID: "snap-2",
|
||||
CreatedAt: time.Date(2026, 10, 30, 1, 0, 0, 0, time.UTC),
|
||||
Rows: []WriteSetRow{
|
||||
{Table: "resources", ID: "res-reused", Existing: true, Owner: "10001/", Reused: true, ReuseEvidence: "digest-1"},
|
||||
{Table: "resources", ID: "res-new"},
|
||||
},
|
||||
Universe: map[string]TableFingerprint{"resources": {Count: 3, Digest: "abc"}}}
|
||||
if err := set.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := set.Verify(); err != nil {
|
||||
t.Fatalf("复用行必须能通过校验: %v", err)
|
||||
}
|
||||
if created := set.CreatedIDs("resources"); len(created) != 1 || created[0] != "res-new" {
|
||||
t.Fatalf("复用行被当成本次新建(恢复会删掉它):%v", created)
|
||||
}
|
||||
if reused := set.ReusedIDs("resources"); len(reused) != 1 || reused[0] != "res-reused" {
|
||||
t.Fatalf("reused resources = %v", reused)
|
||||
}
|
||||
|
||||
noEvidence := set
|
||||
noEvidence.Rows = append([]WriteSetRow{}, set.Rows...)
|
||||
for index := range noEvidence.Rows {
|
||||
if noEvidence.Rows[index].ID == "res-reused" {
|
||||
noEvidence.Rows[index].ReuseEvidence = ""
|
||||
}
|
||||
}
|
||||
if err := noEvidence.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := noEvidence.Verify(); err == nil {
|
||||
t.Fatal("复用行没有证据摘要时必须校验失败")
|
||||
}
|
||||
|
||||
notExisting := set
|
||||
notExisting.Rows = append([]WriteSetRow{}, set.Rows...)
|
||||
for index := range notExisting.Rows {
|
||||
if notExisting.Rows[index].ID == "res-reused" {
|
||||
notExisting.Rows[index].Existing = false
|
||||
}
|
||||
}
|
||||
if err := notExisting.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := notExisting.Verify(); err == nil {
|
||||
t.Fatal("复用行必须同时是写前已存在行(否则会被恢复当残渣)")
|
||||
}
|
||||
|
||||
// 旧记录(没有复用字段)仍然必须能读回:格式演进不能把历史台账判成损坏。
|
||||
legacy := sampleWriteSet()
|
||||
if err := legacy.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := legacy.Verify(); err != nil {
|
||||
t.Fatalf("历史写集合记录必须继续有效: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +102,50 @@ FROM library_jobs WHERE id=$1`, id).Scan(&job.ID, &job.Scope, &kind, &state, &jo
|
||||
return job, nil
|
||||
}
|
||||
|
||||
// LibraryReuseSource 是一条"这个目标的持久来源"记录:成功同步过的快照,
|
||||
// 或这个目标自己的预备份。资源复用只认它作来源证据。
|
||||
type LibraryReuseSource struct {
|
||||
JobID string
|
||||
SnapshotID string
|
||||
Kind string // sync | prebackup
|
||||
FinishedAt time.Time
|
||||
}
|
||||
|
||||
// LibraryReuseSources 读这个目标的持久来源(含已被恢复顶替的生效版本)。
|
||||
// 同步来源要求同时存在 production_effective 版本登记:只写成功但版本没登记的
|
||||
// 作业不算可审计来源。
|
||||
func (s *Store) LibraryReuseSources(ctx context.Context, scope string, limit int) ([]LibraryReuseSource, error) {
|
||||
if limit < 1 || limit > 200 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT j.id, j.target_snapshot_id, 'sync', coalesce(j.finished_at, j.updated_at)
|
||||
FROM library_jobs j
|
||||
WHERE j.scope=$1 AND j.kind='sync' AND j.state='succeeded' AND coalesce(j.target_snapshot_id,'')<>''
|
||||
AND EXISTS (SELECT 1 FROM library_versions v
|
||||
WHERE v.scope=j.scope AND v.kind='production_effective'
|
||||
AND v.snapshot_id=j.target_snapshot_id AND v.source_job_id=j.id)
|
||||
UNION ALL
|
||||
SELECT coalesce(v.source_job_id,''), v.snapshot_id, 'prebackup', v.created_at
|
||||
FROM library_versions v
|
||||
WHERE v.scope=$1 AND v.kind='production_prebackup' AND coalesce(v.snapshot_id,'')<>''
|
||||
ORDER BY 4 DESC
|
||||
LIMIT $2`, scope, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load library reuse sources: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
sources := make([]LibraryReuseSource, 0, 8)
|
||||
for rows.Next() {
|
||||
var source LibraryReuseSource
|
||||
if err := rows.Scan(&source.JobID, &source.SnapshotID, &source.Kind, &source.FinishedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan library reuse source: %w", err)
|
||||
}
|
||||
sources = append(sources, source)
|
||||
}
|
||||
return sources, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) libraryStages(ctx context.Context, id string) ([]governance.LibraryStage, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT stage, state, total, completed, bytes_total, bytes_done, detail, updated_at
|
||||
|
||||
@@ -94,6 +94,9 @@ type ProductionLibraryTarget struct {
|
||||
snapshotKB string
|
||||
// targetEmbeddingModel 是目标知识库绑定的 embedding 模型 id(文档行要换成它)。
|
||||
targetEmbeddingModel string
|
||||
// reusedResources 是"写前已存在、且已证明可复用"的资源身份:写入阶段跳过它们,
|
||||
// 既不重写行也不重传对象。PlanWriteSet 与 Apply 在同一个作业里串行执行。
|
||||
reusedResources map[string]bool
|
||||
}
|
||||
|
||||
// resolveStorageBackend 在目标库里找本次写入的存储后端。找不到就报错:宁可不写,
|
||||
@@ -383,6 +386,14 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
// 已证明与目标现状完全一致的资源:不重写这一行(连 updated_at 都不碰)。
|
||||
// 它在写集合里是 Existing+Reused,恢复清理也不会看它。
|
||||
if table == resourceReuseTableName && t.reusedResources != nil {
|
||||
var identity string
|
||||
if json.Unmarshal(row["id"], &identity) == nil && t.reusedResources[identity] {
|
||||
continue
|
||||
}
|
||||
}
|
||||
// 行必须先映射到目标范围再落库:原样插入会把行写进来源租户/知识库,
|
||||
// 目标范围的删除、校验与恢复都看不见它们。
|
||||
if err := t.retargetRow(table, row); err != nil {
|
||||
@@ -465,6 +476,11 @@ func (t *ProductionLibraryTarget) importResources(ctx context.Context, archive *
|
||||
if !ok || location.Path == "" {
|
||||
return fmt.Errorf("生产缺少资源 %s 的存储路径", part.Name)
|
||||
}
|
||||
// 复用资源的对象字节已经在计划阶段逐字节比过:不再重传(重传只会改
|
||||
// 对象的写入时间,没有任何收益)。
|
||||
if t.reusedResources[part.Name] {
|
||||
continue
|
||||
}
|
||||
reader, err := archive.OpenPart(part)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// 资源复用证据:目标里已经有这个资源 ID、但它不在预备份范围内时,唯一安全的
|
||||
// 处理是证明"本次要写的那一行/那一份对象字节,已经原样存在于目标里"。
|
||||
//
|
||||
// 为什么不能按"没人引用"推断归属:历史孤儿、同租户另一个知识库暂未绑定的资源
|
||||
// 同样没人引用;按它放行等于允许覆盖别人的数据。这里的证据必须同时满足三件事:
|
||||
// 1. 有可审计的持久来源(治理库里这个目标成功同步过的快照或它自己的预备份,
|
||||
// 且同一资源行内容一致);
|
||||
// 2. 整行内容(除易变时间戳外)与本次写入将要落地的行完全一致;
|
||||
// 3. 该资源的对象字节与本次快照归档摘要一致。
|
||||
//
|
||||
// 证据独立落盘(reuse/<digest>.json),摘要写进写集合行;它不冒充历史写集合——
|
||||
// 那两次没有 writeset 的成功作业依然没有 writeset。
|
||||
|
||||
const (
|
||||
reuseDirName = "reuse"
|
||||
reuseEvidenceVersion = 1
|
||||
resourceReuseTableName = "resources"
|
||||
)
|
||||
|
||||
// reuseObject 是一个资源对象的字节证据:归档摘要 + 目标现状必须匹配。
|
||||
type reuseObject struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
// ResourceReuseEvidence 是"这一行可以复用"的完整证据文档。
|
||||
// Digest 覆盖除自己以外的全部内容:被改动就对不上写集合行里的摘要。
|
||||
type ResourceReuseEvidence struct {
|
||||
Version int `json:"version"`
|
||||
JobID string `json:"job_id"`
|
||||
Target string `json:"target"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
SourceTenantID uint64 `json:"source_tenant_id"`
|
||||
SourceKnowledgeBaseID string `json:"source_knowledge_base_id"`
|
||||
TargetSnapshotID string `json:"target_snapshot_id"`
|
||||
ResourceID string `json:"resource_id"`
|
||||
// ArchivedRowDigest 是来源快照里那一行(不含易变时间戳)的摘要。
|
||||
ArchivedRowDigest string `json:"archived_row_digest"`
|
||||
// RetargetedRowDigest 是同一行按本次目标范围/存储改写后的摘要。
|
||||
RetargetedRowDigest string `json:"retargeted_row_digest"`
|
||||
// TargetRowDigest 是目标现状按归档列投影后的摘要:必须与上一条一致。
|
||||
TargetRowDigest string `json:"target_row_digest"`
|
||||
// ProvenanceJobID / ProvenanceSnapshotID 指向一次**已成功**的同步作业及
|
||||
// 它当时写入目标的快照;ProvenanceRowDigest 是那份快照里同一行的摘要。
|
||||
ProvenanceJobID string `json:"provenance_job_id"`
|
||||
ProvenanceSnapshotID string `json:"provenance_snapshot_id"`
|
||||
ProvenanceKind string `json:"provenance_kind"`
|
||||
ProvenanceRowDigest string `json:"provenance_row_digest"`
|
||||
Objects []reuseObject `json:"objects,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
func reusePath(archiveRoot, digest string) string {
|
||||
return filepath.Join(archiveRoot, reuseDirName, digest+".json")
|
||||
}
|
||||
|
||||
// Seal 规范化并写入摘要;字段顺序由 json.Marshal 的结构体顺序固定。
|
||||
func (e *ResourceReuseEvidence) Seal() error {
|
||||
copied := *e
|
||||
copied.Digest = ""
|
||||
raw, err := json.Marshal(copied)
|
||||
if err != nil {
|
||||
return fmt.Errorf("编码复用证据失败: %w", err)
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
e.Digest = hex.EncodeToString(sum[:])
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify 校验版本、必备身份与摘要;缺任何一项都不许拿它去放行覆盖。
|
||||
func (e *ResourceReuseEvidence) Verify() error {
|
||||
if e.Version != reuseEvidenceVersion {
|
||||
return fmt.Errorf("复用证据版本 %d 不受支持(当前 %d)", e.Version, reuseEvidenceVersion)
|
||||
}
|
||||
if e.JobID == "" || e.Target == "" || e.TargetSnapshotID == "" || e.ResourceID == "" ||
|
||||
e.ProvenanceJobID == "" || e.ProvenanceSnapshotID == "" || e.ProvenanceKind == "" {
|
||||
return errors.New("复用证据缺少作业/目标/快照/来源身份")
|
||||
}
|
||||
if e.TenantID == 0 || e.KnowledgeBaseID == "" || e.SourceTenantID == 0 || e.SourceKnowledgeBaseID == "" {
|
||||
return errors.New("复用证据缺少范围身份")
|
||||
}
|
||||
// 三份行摘要必须一致:来源行、映射后的行、目标现状是同一份内容。
|
||||
if e.RetargetedRowDigest == "" || e.TargetRowDigest != e.RetargetedRowDigest {
|
||||
return errors.New("复用证据里目标行与本次要写的行不一致")
|
||||
}
|
||||
if e.ProvenanceRowDigest != e.ArchivedRowDigest {
|
||||
return errors.New("复用证据里的来源行与本次快照行不一致")
|
||||
}
|
||||
copied := *e
|
||||
copied.Digest = ""
|
||||
raw, err := json.Marshal(copied)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
if hex.EncodeToString(sum[:]) != e.Digest {
|
||||
return errors.New("复用证据摘要不符(内容被改动或截断)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PersistReuseEvidence 按内容寻址落盘(只读、幂等)。同摘要已存在时校验后直接返回。
|
||||
func PersistReuseEvidence(archiveRoot string, evidence *ResourceReuseEvidence) (string, error) {
|
||||
if evidence.Digest == "" {
|
||||
if err := evidence.Seal(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if evidence.Digest == "" {
|
||||
return "", errors.New("复用证据没有摘要,拒绝落盘")
|
||||
}
|
||||
directory := filepath.Join(archiveRoot, reuseDirName)
|
||||
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||
return "", fmt.Errorf("创建复用证据目录失败: %w", err)
|
||||
}
|
||||
path := reusePath(archiveRoot, evidence.Digest)
|
||||
if existing, readErr := os.ReadFile(path); readErr == nil {
|
||||
var recorded ResourceReuseEvidence
|
||||
if json.Unmarshal(existing, &recorded) != nil || recorded.Verify() != nil || recorded.Digest != evidence.Digest {
|
||||
return "", errors.New("复用证据同名但内容不符,拒绝覆盖")
|
||||
}
|
||||
return path, nil
|
||||
} else if !errors.Is(readErr, os.ErrNotExist) {
|
||||
return "", fmt.Errorf("读取已有复用证据失败: %w", readErr)
|
||||
}
|
||||
raw, err := json.Marshal(*evidence)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("编码复用证据失败: %w", err)
|
||||
}
|
||||
temporary := path + ".tmp"
|
||||
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("写入复用证据失败: %w", err)
|
||||
}
|
||||
if _, err := file.Write(raw); err != nil {
|
||||
file.Close()
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("写入复用证据失败: %w", err)
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
file.Close()
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("复用证据未落盘: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("复用证据未落盘: %w", err)
|
||||
}
|
||||
if err := os.Chmod(temporary, 0444); err != nil {
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("复用证据只读化失败: %w", err)
|
||||
}
|
||||
if err := os.Rename(temporary, path); err != nil {
|
||||
os.Remove(temporary)
|
||||
return "", fmt.Errorf("复用证据落盘失败: %w", err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// LoadReuseEvidence 取回并校验证据;摘要或绑定对不上就失败。
|
||||
func LoadReuseEvidence(archiveRoot, digest string) (*ResourceReuseEvidence, error) {
|
||||
if digest == "" {
|
||||
return nil, errors.New("没有复用证据摘要")
|
||||
}
|
||||
raw, err := os.ReadFile(reusePath(archiveRoot, digest))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("复用证据缺失(%s)", digest)
|
||||
}
|
||||
var evidence ResourceReuseEvidence
|
||||
if err := json.Unmarshal(raw, &evidence); err != nil {
|
||||
return nil, errors.New("复用证据不可解析")
|
||||
}
|
||||
if err := evidence.Verify(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if evidence.Digest != digest {
|
||||
return nil, errors.New("复用证据摘要不符")
|
||||
}
|
||||
return &evidence, nil
|
||||
}
|
||||
|
||||
// archivedRowByID 从归档部件里读出指定身份的行(原样,不改写)。
|
||||
func archivedRowByID(archive *librarysnapshot.Archive, part librarysnapshot.Part, id string) (map[string]json.RawMessage, error) {
|
||||
reader, err := archive.OpenPart(part)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取归档 %s 失败", part.Name)
|
||||
}
|
||||
defer reader.Close()
|
||||
decoder := json.NewDecoder(reader)
|
||||
token, err := decoder.Token()
|
||||
if err != nil || token != json.Delim('[') {
|
||||
return nil, fmt.Errorf("归档 %s 不是行数组", part.Name)
|
||||
}
|
||||
for decoder.More() {
|
||||
var row map[string]json.RawMessage
|
||||
if err := decoder.Decode(&row); err != nil {
|
||||
return nil, fmt.Errorf("解析归档 %s 行失败", part.Name)
|
||||
}
|
||||
var identity string
|
||||
if json.Unmarshal(row["id"], &identity) == nil && identity == id {
|
||||
return row, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// targetResourceRow 读出目标里这一行;不存在返回 nil。
|
||||
func (t *ProductionLibraryTarget) targetResourceRow(ctx context.Context, id string) (map[string]json.RawMessage, error) {
|
||||
var raw map[string]json.RawMessage
|
||||
err := t.pool.QueryRow(ctx, `SELECT to_jsonb(r) FROM resources r WHERE r.id=$1 AND r.tenant_id=$2`,
|
||||
id, t.scope.TenantID).Scan(&raw)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("读取目标资源 %s 失败", id)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// cloneRow 复制一行,避免改写归档缓存/调用方数据。
|
||||
func cloneRow(row map[string]json.RawMessage) map[string]json.RawMessage {
|
||||
copied := make(map[string]json.RawMessage, len(row))
|
||||
for key, value := range row {
|
||||
copied[key] = value
|
||||
}
|
||||
return copied
|
||||
}
|
||||
|
||||
// projectRow 只保留给定列集合里的列:目标行比归档多出的默认列不参与比对。
|
||||
func projectRow(row map[string]json.RawMessage, columns map[string]bool) map[string]json.RawMessage {
|
||||
projected := make(map[string]json.RawMessage, len(columns))
|
||||
for key := range columns {
|
||||
if value, ok := row[key]; ok {
|
||||
projected[key] = value
|
||||
}
|
||||
}
|
||||
return projected
|
||||
}
|
||||
|
||||
// targetObjectSHA256 读取目标对象的实际字节并返回大小与摘要。
|
||||
func (t *ProductionLibraryTarget) targetObjectSHA256(ctx context.Context, provider, path string) (int64, string, error) {
|
||||
if t.config.ObjectsReader == nil {
|
||||
return 0, "", errors.New("缺少目标对象读取器,无法证明对象字节一致")
|
||||
}
|
||||
reader, err := t.config.ObjectsReader.Open(ctx, provider, path)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
defer reader.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(hash, reader)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return size, hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// resourceLocatorField 解析改写后的资源行里的定位符与提供方(与写入/校验同一口径)。
|
||||
func resourceLocatorField(row map[string]json.RawMessage) (provider, path string, err error) {
|
||||
if err := json.Unmarshal(row["provider"], &provider); err != nil {
|
||||
return "", "", errors.New("资源行提供方不可解析")
|
||||
}
|
||||
if err := json.Unmarshal(row["physical_path"], &path); err != nil || path == "" {
|
||||
return "", "", errors.New("资源行定位符不可解析")
|
||||
}
|
||||
return provider, path, nil
|
||||
}
|
||||
|
||||
// provenanceFor 在"这个目标的持久来源"里找同一资源的同一行内容:
|
||||
// - 成功同步过的快照(说明这个目标先前就把这行写进去过);
|
||||
// - 这个目标自己的预备份(说明这行本来就是本目标历史状态的一部分)。
|
||||
//
|
||||
// 找不到就说明归属无法证明:即使目标里那行看起来一模一样,也不能覆盖/复用。
|
||||
func provenanceFor(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, targetSnapshotID,
|
||||
id, archivedDigest string, prior []libraryrelease.PriorSnapshot) (libraryrelease.PriorSnapshot, string, error) {
|
||||
for _, candidate := range prior {
|
||||
if candidate.JobID == "" || candidate.SnapshotID == "" {
|
||||
continue
|
||||
}
|
||||
if candidate.SnapshotID == targetSnapshotID {
|
||||
// 本次要写的快照本身就是一次成功同步写进目标的那一份。
|
||||
return candidate, archivedDigest, nil
|
||||
}
|
||||
priorManifest, err := archive.Load(candidate.SnapshotID)
|
||||
if err != nil {
|
||||
// 来源快照不可读不能当作"没有来源":那会把不可审计变成静默放行。
|
||||
return libraryrelease.PriorSnapshot{}, "", fmt.Errorf("复用来源快照 %s 不可读", candidate.SnapshotID)
|
||||
}
|
||||
if priorManifest.TenantID != manifest.TenantID || priorManifest.KnowledgeBaseID != manifest.KnowledgeBaseID {
|
||||
continue
|
||||
}
|
||||
var part librarysnapshot.Part
|
||||
found := false
|
||||
for _, item := range priorManifest.Parts {
|
||||
if item.Kind == "database" && item.Name == resourceReuseTableName {
|
||||
part, found = item, true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
row, err := archivedRowByID(archive, part, id)
|
||||
if err != nil {
|
||||
return libraryrelease.PriorSnapshot{}, "", err
|
||||
}
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
digest, err := canonicalRowDigest(row)
|
||||
if err != nil {
|
||||
return libraryrelease.PriorSnapshot{}, "", err
|
||||
}
|
||||
if digest != archivedDigest {
|
||||
continue
|
||||
}
|
||||
return candidate, digest, nil
|
||||
}
|
||||
return libraryrelease.PriorSnapshot{}, "", nil
|
||||
}
|
||||
|
||||
// planReusableResource 是资源冲突的唯一放行路径:内容相同 + 字节相同 + 来源可审计。
|
||||
// 返回写集合行(Existing + Reused)并把它登记到本次目标,写入阶段据此跳过这一行。
|
||||
func (t *ProductionLibraryTarget) planReusableResource(ctx context.Context, archive *librarysnapshot.Archive,
|
||||
manifest librarysnapshot.Manifest, part librarysnapshot.Part, id, owner string,
|
||||
plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSetRow, error) {
|
||||
archived, err := archivedRowByID(archive, part, id)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSetRow{}, err
|
||||
}
|
||||
if archived == nil {
|
||||
return libraryrelease.WriteSetRow{}, fmt.Errorf("资源 %s 在快照 %s 的 resources 部件里找不到", id, plan.TargetSnapshotID)
|
||||
}
|
||||
archivedDigest, err := canonicalRowDigest(archived)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSetRow{}, err
|
||||
}
|
||||
columns := map[string]bool{}
|
||||
for key := range archived {
|
||||
columns[key] = true
|
||||
}
|
||||
retargeted := cloneRow(archived)
|
||||
if err := t.retargetRow(resourceReuseTableName, retargeted); err != nil {
|
||||
return libraryrelease.WriteSetRow{}, err
|
||||
}
|
||||
retargetedDigest, err := canonicalRowDigest(retargeted)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSetRow{}, err
|
||||
}
|
||||
existing, err := t.targetResourceRow(ctx, id)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSetRow{}, err
|
||||
}
|
||||
if existing == nil {
|
||||
return libraryrelease.WriteSetRow{}, fmt.Errorf("资源 %s 在目标里已不存在,不能复用", id)
|
||||
}
|
||||
targetDigest, err := canonicalRowDigest(projectRow(existing, columns))
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSetRow{}, err
|
||||
}
|
||||
if targetDigest != retargetedDigest {
|
||||
return libraryrelease.WriteSetRow{}, fmt.Errorf(
|
||||
"资源 ID 冲突:%s 已存在于目标租户但内容与本次快照不一致(目标 %s / 本次 %s),拒绝覆盖",
|
||||
id, shortDigest(targetDigest), shortDigest(retargetedDigest))
|
||||
}
|
||||
provenance, provenanceDigest, err := provenanceFor(archive, manifest, plan.TargetSnapshotID, id, archivedDigest, plan.PriorSnapshots)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSetRow{}, err
|
||||
}
|
||||
if provenance.JobID == "" {
|
||||
return libraryrelease.WriteSetRow{}, fmt.Errorf(
|
||||
"资源 ID 冲突:%s 已存在于目标租户但不在预备份范围内,且没有可审计的持久来源(成功同步/预备份)证明它属于本目标,拒绝覆盖", id)
|
||||
}
|
||||
provider, path, err := resourceLocatorField(retargeted)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSetRow{}, err
|
||||
}
|
||||
objects := make([]reuseObject, 0, 2)
|
||||
for _, item := range manifest.Parts {
|
||||
if item.Kind != "resource" || item.Name != id {
|
||||
continue
|
||||
}
|
||||
size, digest, err := t.targetObjectSHA256(ctx, provider, path)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSetRow{}, fmt.Errorf("资源 %s 的对象读不到,无法证明字节一致:%w", id, err)
|
||||
}
|
||||
if size != item.Bytes || digest != item.SHA256 {
|
||||
return libraryrelease.WriteSetRow{}, fmt.Errorf(
|
||||
"资源 ID 冲突:%s 的目标对象字节与本次快照不一致(%d/%d 字节),拒绝覆盖",
|
||||
id, size, item.Bytes)
|
||||
}
|
||||
objects = append(objects, reuseObject{Name: item.Name, Path: path, Bytes: size, SHA256: digest})
|
||||
}
|
||||
evidence := &ResourceReuseEvidence{
|
||||
Version: reuseEvidenceVersion, JobID: plan.JobID, Target: plan.Target,
|
||||
TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID,
|
||||
SourceTenantID: manifest.TenantID, SourceKnowledgeBaseID: manifest.KnowledgeBaseID,
|
||||
TargetSnapshotID: plan.TargetSnapshotID, ResourceID: id,
|
||||
ArchivedRowDigest: archivedDigest, RetargetedRowDigest: retargetedDigest, TargetRowDigest: targetDigest,
|
||||
ProvenanceJobID: provenance.JobID, ProvenanceSnapshotID: provenance.SnapshotID, ProvenanceKind: provenance.Kind,
|
||||
ProvenanceRowDigest: provenanceDigest, Objects: objects, CreatedAt: t.now().UTC(),
|
||||
}
|
||||
if _, err := PersistReuseEvidence(t.config.ArchiveRoot, evidence); err != nil {
|
||||
return libraryrelease.WriteSetRow{}, fmt.Errorf("复用证据无法落盘,拒绝放行覆盖:%w", err)
|
||||
}
|
||||
if t.reusedResources == nil {
|
||||
t.reusedResources = map[string]bool{}
|
||||
}
|
||||
t.reusedResources[id] = true
|
||||
return libraryrelease.WriteSetRow{Table: resourceReuseTableName, ID: id, Existing: true,
|
||||
Owner: owner, Reused: true, ReuseEvidence: evidence.Digest}, nil
|
||||
}
|
||||
|
||||
func shortDigest(digest string) string {
|
||||
if len(digest) > 12 {
|
||||
return digest[:12]
|
||||
}
|
||||
return digest
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
// 这一组用例覆盖真实缺陷现场:同步 A 成功 → 手工恢复 B(对象保留、资源行不删)
|
||||
// → 再同步 A。旧预检把"恢复保留下来的、本目标自己上次写进去的资源"判成
|
||||
// "其它知识库的同 ID 冲突",整个同步在写第一笔之前被拒。
|
||||
//
|
||||
// 复用的放行条件必须是可审计的证据,而不是"目标里没人引用":
|
||||
// - 目标现状那一行与本次要写的行逐列相同(除易变时间戳);
|
||||
// - 该资源的对象字节与本次快照归档摘要一致;
|
||||
// - 治理库里存在"这个目标成功同步过、且含有同一行内容"的快照/作业。
|
||||
// 缺任何一条都必须继续拒绝,且拒绝发生在写入之前。
|
||||
|
||||
const (
|
||||
wsHandleNew = "gggggggggggggggggggggg"
|
||||
// resNewBody 是"A 的下一次发布"新增资源的内容;与 res-src 的 "12345678" 不同,
|
||||
// 这样"本次真新建"和"证明可复用"在同一份写集合里能同时被观察到。
|
||||
resNewBody = "87654321"
|
||||
)
|
||||
|
||||
// reusePriors 是治理库会给出的来源清单:一次成功同步 + 它产生的预备份。
|
||||
func reusePriors(syncJobID, syncSnapshotID string, prebackups ...string) []libraryrelease.PriorSnapshot {
|
||||
priors := []libraryrelease.PriorSnapshot{{JobID: syncJobID, SnapshotID: syncSnapshotID, Kind: libraryrelease.PriorSnapshotSync}}
|
||||
for _, snapshot := range prebackups {
|
||||
priors = append(priors, libraryrelease.PriorSnapshot{JobID: syncJobID, SnapshotID: snapshot,
|
||||
Kind: libraryrelease.PriorSnapshotPrebackup})
|
||||
}
|
||||
return priors
|
||||
}
|
||||
|
||||
func requestForWithPrior(snapshotID, jobID string, prior []libraryrelease.PriorSnapshot) libraryrelease.Request {
|
||||
request := requestFor(snapshotID)
|
||||
request.JobID = jobID
|
||||
request.PriorSnapshots = prior
|
||||
return request
|
||||
}
|
||||
|
||||
// syncThenRestore 造出现场:同步 A 成功,然后按生产恢复作业的方式恢复到 A 的
|
||||
// 预备份(再走一次 Run:新的预备份 + 它自己的写集合)。恢复不删 resources 表,
|
||||
// 所以 A 写进去的资源会作为"没人引用的行"留在目标里——正是本次缺陷的前提。
|
||||
func syncThenRestore(t *testing.T, fx *writeSetFixture, snapshotID string) (*ProductionLibraryTarget,
|
||||
*ProductionBackupStore, libraryrelease.Outcome, []libraryrelease.PriorSnapshot) {
|
||||
t.Helper()
|
||||
target := fx.target(t)
|
||||
backups := fx.backups(t, target)
|
||||
first := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a", nil), target, backups, nil)
|
||||
if first.State != "succeeded" {
|
||||
t.Fatalf("首次同步 A 必须成功:%+v", first)
|
||||
}
|
||||
if first.Backup.WriteSet == nil {
|
||||
t.Fatal("首次同步必须落盘写集合")
|
||||
}
|
||||
// 恢复作业同样只认已登记来源:它自己刚拍的那份预备份就是本目标的历史状态。
|
||||
restore := libraryrelease.Run(fx.ctx, requestForWithPrior(first.Backup.SnapshotID, "job-restore-b",
|
||||
reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)), target, backups, nil)
|
||||
if restore.State != "succeeded" {
|
||||
t.Fatalf("手工恢复 B 必须成功:%+v", restore)
|
||||
}
|
||||
var bindings int64
|
||||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resource_bindings WHERE resource_id='res-src'`).Scan(&bindings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bindings != 0 {
|
||||
t.Fatalf("恢复后 res-src 不应还有绑定行(现场前提):%d", bindings)
|
||||
}
|
||||
return target, backups, first, reusePriors(first.Backup.WriteSet.JobID, snapshotID, first.Backup.SnapshotID)
|
||||
}
|
||||
|
||||
// resourceFingerprint 取 PG 行的 xmin 与整行 JSON:xmin 不变才能证明"没有重写这一行"。
|
||||
func resourceFingerprint(t *testing.T, pool *pgxpool.Pool, id string) (string, string) {
|
||||
t.Helper()
|
||||
var xmin, row string
|
||||
if err := pool.QueryRow(t.Context(), `SELECT xmin::text, to_jsonb(r)::text FROM resources r WHERE id=$1`, id).
|
||||
Scan(&xmin, &row); err != nil {
|
||||
t.Fatalf("读取资源 %s: %v", id, err)
|
||||
}
|
||||
return xmin, row
|
||||
}
|
||||
|
||||
func resourceRowOf(t *testing.T, pool *pgxpool.Pool, id string) string {
|
||||
t.Helper()
|
||||
var row string
|
||||
if err := pool.QueryRow(t.Context(), `SELECT to_jsonb(r)::text FROM resources r WHERE id=$1`, id).Scan(&row); err != nil {
|
||||
t.Fatalf("读取资源 %s: %v", id, err)
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func TestLibraryResyncAfterRestoreReusesProvenResourcesIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
target, backups, first, priors := syncThenRestore(t, fx, snapshotID)
|
||||
|
||||
beforeXmin, beforeRow := resourceFingerprint(t, fx.pool, "res-src")
|
||||
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
|
||||
otherBefore := resourceRowOf(t, fx.pool, "res-other")
|
||||
sharedBefore := resourceRowOf(t, fx.pool, "res-shared")
|
||||
foreignBefore := resourceRowOf(t, fx.pool, "res-foreign")
|
||||
|
||||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-a-again", priors),
|
||||
target, backups, nil)
|
||||
if outcome.State != "succeeded" {
|
||||
t.Fatalf("恢复后再次同步 A 必须完成:%+v", outcome)
|
||||
}
|
||||
if outcome.Backup.WriteSet == nil {
|
||||
t.Fatal("写集合必须落盘")
|
||||
}
|
||||
ledger, err := LoadWriteSet(fx.archiveRoot, outcome.Backup.WriteSet.Digest)
|
||||
if err != nil {
|
||||
t.Fatalf("写集合记录取不回来:%v", err)
|
||||
}
|
||||
reused := ledger.ReusedIDs("resources")
|
||||
if len(reused) != 1 || reused[0] != "res-src" {
|
||||
t.Fatalf("复用资源 = %v,want [res-src]", reused)
|
||||
}
|
||||
if created := ledger.CreatedIDs("resources"); len(created) != 0 {
|
||||
t.Fatalf("复用资源不能被记成本次新建(恢复会删掉它):%v", created)
|
||||
}
|
||||
var reuseRow libraryrelease.WriteSetRow
|
||||
for _, row := range ledger.Rows {
|
||||
if row.Table == "resources" && row.ID == "res-src" {
|
||||
reuseRow = row
|
||||
}
|
||||
}
|
||||
if !reuseRow.Existing || !reuseRow.Reused || reuseRow.ReuseEvidence == "" {
|
||||
t.Fatalf("复用行必须带证据摘要并标记为写前已存在:%+v", reuseRow)
|
||||
}
|
||||
evidence, err := LoadReuseEvidence(fx.archiveRoot, reuseRow.ReuseEvidence)
|
||||
if err != nil {
|
||||
t.Fatalf("复用证据取不回来:%v", err)
|
||||
}
|
||||
if evidence.ProvenanceSnapshotID != snapshotID || evidence.ProvenanceJobID != first.Backup.WriteSet.JobID {
|
||||
t.Fatalf("复用证据没有指向前一次成功同步:%+v", evidence)
|
||||
}
|
||||
if evidence.TargetRowDigest != evidence.RetargetedRowDigest || evidence.ProvenanceRowDigest != evidence.ArchivedRowDigest {
|
||||
t.Fatalf("复用证据的行摘要不一致:%+v", evidence)
|
||||
}
|
||||
if len(evidence.Objects) != 1 || evidence.Objects[0].SHA256 == "" {
|
||||
t.Fatalf("复用证据必须逐个记录对象字节:%+v", evidence.Objects)
|
||||
}
|
||||
|
||||
// 复用不许改原行:xmin 与整行 JSON 都必须原样。
|
||||
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
|
||||
if afterXmin != beforeXmin {
|
||||
t.Fatalf("复用资源被重写了:xmin %s -> %s", beforeXmin, afterXmin)
|
||||
}
|
||||
if afterRow != beforeRow {
|
||||
t.Fatalf("复用资源内容变了:\n%s\n->\n%s", beforeRow, afterRow)
|
||||
}
|
||||
// 历史孤儿/其它知识库/共享/外租户资源一律不许被覆盖。
|
||||
for id, want := range map[string]string{"res-orphan": orphanBefore, "res-other": otherBefore,
|
||||
"res-shared": sharedBefore, "res-foreign": foreignBefore} {
|
||||
if got := resourceRowOf(t, fx.pool, id); got != want {
|
||||
t.Fatalf("资源 %s 被改动:\n%s\n->\n%s", id, want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryResyncAfterRestoreRejectsReuseWithoutProvenanceIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
target, backups, first, _ := syncThenRestore(t, fx, snapshotID)
|
||||
before := universeDump(t, fx.pool)
|
||||
|
||||
// 已登记的来源里没有 res-src(只有这次同步之前的预备份):即使目标那一行
|
||||
// 看起来一模一样,也不许放行。
|
||||
noSource := []libraryrelease.PriorSnapshot{{JobID: first.Backup.WriteSet.JobID,
|
||||
SnapshotID: first.Backup.SnapshotID, Kind: libraryrelease.PriorSnapshotPrebackup}}
|
||||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-no-provenance", noSource), target, backups, nil)
|
||||
if outcome.State != "failed" {
|
||||
t.Fatalf("没有来源证据时必须拒绝:%+v", outcome)
|
||||
}
|
||||
if outcome.FailedAt != "" {
|
||||
t.Fatalf("冲突必须在任何阶段写入之前被拒绝,failed_at=%s", outcome.FailedAt)
|
||||
}
|
||||
if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "可审计") {
|
||||
t.Fatalf("拒绝原因必须点名资源并说明缺来源证据:%s", outcome.Error)
|
||||
}
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryResyncAfterRestoreRejectsReuseWhenContentDiffersIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
|
||||
// 同 ID 同租户,但内容被改过(这里改大小,等价于"另一份文件占了这个 ID")。
|
||||
if _, err := fx.pool.Exec(fx.ctx, `UPDATE resources SET size=size+1 WHERE id='res-src'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changed := resourceRowOf(t, fx.pool, "res-src")
|
||||
before := universeDump(t, fx.pool)
|
||||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(snapshotID, "job-sync-content-diff", priors), target, backups, nil)
|
||||
if outcome.State != "failed" {
|
||||
t.Fatalf("内容不一致必须拒绝:%+v", outcome)
|
||||
}
|
||||
if !strings.Contains(outcome.Error, "res-src") || !strings.Contains(outcome.Error, "内容") {
|
||||
t.Fatalf("拒绝原因必须点名内容不一致:%s", outcome.Error)
|
||||
}
|
||||
if resourceRowOf(t, fx.pool, "res-src") != changed {
|
||||
t.Fatal("被拒绝的同步不许改目标资源")
|
||||
}
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
// 同租户、同 ID、同内容,但来源快照属于另一个知识库:仍然不许复用。
|
||||
// "归属无法证明"包括"来源不同",不能因为字节一样就把别人的行当成自己的。
|
||||
func TestLibraryReuseRequiresSameSourceScopeIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
|
||||
otherSource := sealSourceSnapshotScope(t, fx, wsSourceTenant+1, "kb-source-other", nil, nil)
|
||||
before := universeDump(t, fx.pool)
|
||||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(otherSource, "job-sync-other-source", priors), target, backups, nil)
|
||||
if outcome.State != "failed" {
|
||||
t.Fatalf("来源范围不同必须拒绝复用:%+v", outcome)
|
||||
}
|
||||
if !strings.Contains(outcome.Error, "可审计") {
|
||||
t.Fatalf("拒绝原因必须说明来源不可审计:%s", outcome.Error)
|
||||
}
|
||||
if diff := diffUniverse(before, universeDump(t, fx.pool)); diff != "" {
|
||||
t.Fatalf("被拒绝的同步不许动目标:%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
// 部分写失败:database/resources 已提交、wiki 因重复 slug 失败 → 回滚只清"本次真新建",
|
||||
// 复用资源与历史孤儿原内容完整保留(xmin 也不变)。
|
||||
func TestLibraryHalfWriteRollbackKeepsReusedResourcesIsolated(t *testing.T) {
|
||||
fx := openWriteSetFixture(t)
|
||||
seedTargetData(t, fx)
|
||||
snapshotID := sealSourceSnapshot(t, fx, nil)
|
||||
target, backups, _, priors := syncThenRestore(t, fx, snapshotID)
|
||||
reuseXmin, reuseRow := resourceFingerprint(t, fx.pool, "res-src")
|
||||
orphanBefore := resourceRowOf(t, fx.pool, "res-orphan")
|
||||
|
||||
next := sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB,
|
||||
func(parts map[string][]map[string]any) {
|
||||
parts["chunks"] = append(parts["chunks"], map[string]any{"id": "chunk-src-2", "tenant_id": wsSourceTenant,
|
||||
"knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src", "content": "新增正文 resource://" + wsHandleNew})
|
||||
parts["resources"] = append(parts["resources"], map[string]any{"id": "res-new", "handle": wsHandleNew,
|
||||
"tenant_id": wsSourceTenant, "storage_backend_id": "backend-source", "provider": "oss",
|
||||
"physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-new"),
|
||||
"size": int64(len(resNewBody))})
|
||||
parts["resource_bindings"] = append(parts["resource_bindings"], map[string]any{"id": "bind-src-2",
|
||||
"resource_id": "res-new", "tenant_id": wsSourceTenant, "owner_type": "chunk",
|
||||
"owner_id": "chunk-src-2", "relation": "attachment"})
|
||||
// 同一作用域两页同 slug:wiki 的唯一索引让 wiki 阶段失败(真实半写现场)。
|
||||
parts["wiki_pages"] = []map[string]any{
|
||||
{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"},
|
||||
{"id": "wiki-src-2", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "重复 slug"},
|
||||
}
|
||||
}, map[string]string{"res-new": resNewBody})
|
||||
|
||||
outcome := libraryrelease.Run(fx.ctx, requestForWithPrior(next, "job-sync-half-write", priors), target, backups, nil)
|
||||
if outcome.State != "rolled_back" {
|
||||
t.Fatalf("wiki 半写必须回滚:%+v", outcome)
|
||||
}
|
||||
if outcome.FailedAt != libraryrelease.StageWiki {
|
||||
t.Fatalf("failed_at = %s, want wiki", outcome.FailedAt)
|
||||
}
|
||||
ledger := outcome.Backup.WriteSet
|
||||
if ledger == nil {
|
||||
t.Fatal("半写作业必须留下写集合")
|
||||
}
|
||||
if created := ledger.CreatedIDs("resources"); len(created) != 1 || created[0] != "res-new" {
|
||||
t.Fatalf("本次真新建的资源 = %v,want [res-new]", created)
|
||||
}
|
||||
if reused := ledger.ReusedIDs("resources"); len(reused) != 1 || reused[0] != "res-src" {
|
||||
t.Fatalf("复用资源 = %v,want [res-src]", reused)
|
||||
}
|
||||
// 本次真新建的清干净。
|
||||
var residue int64
|
||||
if err := fx.pool.QueryRow(fx.ctx, `SELECT count(*) FROM resources WHERE id='res-new'`).Scan(&residue); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if residue != 0 {
|
||||
t.Fatal("回滚后本次新建的资源必须被清理")
|
||||
}
|
||||
// 复用资源与历史孤儿原内容(含 xmin)保留。
|
||||
afterXmin, afterRow := resourceFingerprint(t, fx.pool, "res-src")
|
||||
if afterXmin != reuseXmin || afterRow != reuseRow {
|
||||
t.Fatalf("回滚动了复用资源:xmin %s->%s\n%s\n->\n%s", reuseXmin, afterXmin, reuseRow, afterRow)
|
||||
}
|
||||
if resourceRowOf(t, fx.pool, "res-orphan") != orphanBefore {
|
||||
t.Fatal("回滚动了历史孤儿资源")
|
||||
}
|
||||
if err := target.Verify(fx.ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil {
|
||||
t.Fatalf("回滚必须与预备份一致:%v", err)
|
||||
}
|
||||
}
|
||||
@@ -132,6 +132,14 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library
|
||||
if len(plan.Universe) == 0 {
|
||||
return libraryrelease.WriteSet{}, errors.New("预备份没有整实例内容证据,拒绝在没有保护证据的情况下写目标")
|
||||
}
|
||||
// 资源复用的"整行内容"必须按写入时的同一套范围/存储改写来比,所以先把目标
|
||||
// 存储后端解析出来;解析不到就等于写不出可解析的定位符,直接拒绝。
|
||||
if err := t.resolveStorageBackend(ctx); err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
// 每次预检都从空集开始:同一进程里复用目标对象时,上一次作业的"可复用"
|
||||
// 判断不能泄漏到这一次。
|
||||
t.reusedResources = map[string]bool{}
|
||||
archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
|
||||
if err != nil {
|
||||
return libraryrelease.WriteSet{}, errors.New("归档不可用")
|
||||
@@ -173,6 +181,20 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library
|
||||
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id})
|
||||
continue
|
||||
}
|
||||
if part.Name == resourceReuseTableName && !backupResources[id] {
|
||||
// 资源不在预备份范围,但可能正是这个目标上一次成功同步写进去、
|
||||
// 又被恢复保留下来的那份内容。只有"内容/字节相同 + 来源可审计"
|
||||
// 才允许复用(不重写、不清理);否则按跨范围冲突拒绝。
|
||||
if err := t.assertResourceTenant(id, owner); err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
row, reuseErr := t.planReusableResource(ctx, archive, manifest, part, id, owner, plan)
|
||||
if reuseErr != nil {
|
||||
return libraryrelease.WriteSet{}, reuseErr
|
||||
}
|
||||
set.Rows = append(set.Rows, row)
|
||||
continue
|
||||
}
|
||||
if err := t.assertWriteSetOwnership(part.Name, id, owner, backupResources); err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
@@ -265,6 +287,15 @@ WHERE b.id = ANY($1::text[])`
|
||||
return owners, rows.Err()
|
||||
}
|
||||
|
||||
// assertResourceTenant 只检查"同 ID 是否属于别的租户":resources 没有
|
||||
// knowledge_base_id,同租户不等于同知识库,范围问题交给后续的复用证明。
|
||||
func (t *ProductionLibraryTarget) assertResourceTenant(id, owner string) error {
|
||||
if owner != fmt.Sprintf("%d/", t.scope.TenantID) {
|
||||
return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// assertWriteSetOwnership 是跨范围 ID 冲突的预检:同 ID 已经是别人的行时,
|
||||
// 写入会改掉别人的数据(resources 尤其危险:ON CONFLICT 会整行覆盖)。
|
||||
// 宁可整次拒绝,也不许"覆盖一下看起来也能跑"。
|
||||
@@ -272,8 +303,8 @@ func (t *ProductionLibraryTarget) assertWriteSetOwnership(table, id, owner strin
|
||||
target := fmt.Sprintf("%d/%s", t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||||
if table == "resources" {
|
||||
// 资源只有租户:同租户不等于同知识库,必须落在预备份范围内才算自己的。
|
||||
if owner != fmt.Sprintf("%d/", t.scope.TenantID) {
|
||||
return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner)
|
||||
if err := t.assertResourceTenant(id, owner); err != nil {
|
||||
return err
|
||||
}
|
||||
if !backupResources[id] {
|
||||
return fmt.Errorf("资源 ID 冲突:%s 存在于目标租户但不在预备份范围内(属于其它知识库),拒绝覆盖", id)
|
||||
|
||||
@@ -293,6 +293,14 @@ func (fx *writeSetFixture) indexDoc(t *testing.T, id string, tenant uint64, kb,
|
||||
|
||||
// sealSourceSnapshot 造一份"来源范围与目标不同"的整库快照。
|
||||
func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[string][]map[string]any)) string {
|
||||
t.Helper()
|
||||
return sealSourceSnapshotScope(t, fx, wsSourceTenant, wsSourceKB, tweak, nil)
|
||||
}
|
||||
|
||||
// sealSourceSnapshotScope 允许指定来源范围/附加对象:来源快照的身份决定
|
||||
// "同一资源行是否属于同一个来源",复用证据必须按它区分。
|
||||
func sealSourceSnapshotScope(t *testing.T, fx *writeSetFixture, sourceTenant uint64, sourceKB string,
|
||||
tweak func(parts map[string][]map[string]any), extraObjects map[string]string) string {
|
||||
t.Helper()
|
||||
archive, err := librarysnapshot.Open(fx.archiveRoot, 128*1024*1024)
|
||||
if err != nil {
|
||||
@@ -307,20 +315,20 @@ func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[
|
||||
return encoded
|
||||
}
|
||||
parts := map[string][]map[string]any{
|
||||
"knowledges": {{"id": "doc-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||||
"knowledges": {{"id": "doc-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
|
||||
"parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}},
|
||||
"chunks": {{"id": "chunk-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB,
|
||||
"chunks": {{"id": "chunk-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB,
|
||||
"knowledge_id": "doc-src", "content": "来源正文 resource://" + wsHandleSource}},
|
||||
"knowledge_tags": {},
|
||||
"knowledge_tag_relations": {},
|
||||
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": wsSourceTenant,
|
||||
"resources": {{"id": "res-src", "handle": wsHandleSource, "tenant_id": sourceTenant,
|
||||
"storage_backend_id": "backend-source", "provider": "oss",
|
||||
"physical_path": wsLocator("backend-source", "oss", "source-bucket", wsSourceTenant, "res-src"),
|
||||
"physical_path": wsLocator("backend-source", "oss", "source-bucket", sourceTenant, "res-src"),
|
||||
"size": int64(8)}},
|
||||
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": wsSourceTenant,
|
||||
"resource_bindings": {{"id": "bind-src", "resource_id": "res-src", "tenant_id": sourceTenant,
|
||||
"owner_type": "knowledge", "owner_id": "doc-src", "relation": "attachment"}},
|
||||
"wiki_folders": {},
|
||||
"wiki_pages": {{"id": "wiki-src", "tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "slug": "index", "content": "来源 wiki"}},
|
||||
"wiki_pages": {{"id": "wiki-src", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "slug": "index", "content": "来源 wiki"}},
|
||||
"wiki_page_issues": {},
|
||||
"wiki_log_entries": {},
|
||||
}
|
||||
@@ -350,17 +358,38 @@ func sealSourceSnapshot(t *testing.T, fx *writeSetFixture, tweak func(parts map[
|
||||
}
|
||||
resourcePart.Kind, resourcePart.Name, resourcePart.Count = "resource", "res-src", 1
|
||||
manifestParts = append(manifestParts, resourcePart)
|
||||
indexPart, err := archive.Put(bytes.NewReader(encode([]map[string]any{{"id": "chunk-src", "source": map[string]any{
|
||||
"tenant_id": wsSourceTenant, "knowledge_base_id": wsSourceKB, "knowledge_id": "doc-src",
|
||||
"chunk_id": "chunk-src", "source_id": "chunk-src", "content": "来源正文 resource://" + wsHandleSource,
|
||||
"embedding": []float64{0.1, 0.2}}}})))
|
||||
extraNames := make([]string, 0, len(extraObjects))
|
||||
for name := range extraObjects {
|
||||
extraNames = append(extraNames, name)
|
||||
}
|
||||
sort.Strings(extraNames)
|
||||
for _, name := range extraNames {
|
||||
part, err := archive.Put(bytes.NewReader([]byte(extraObjects[name])))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part.Kind, part.Name, part.Count = "resource", name, 1
|
||||
manifestParts = append(manifestParts, part)
|
||||
}
|
||||
// 索引文档按快照里的分块逐条生成:多一个分块就必须多一条向量,否则
|
||||
// 校验会把"写入正确"判成索引缺文档。
|
||||
indexRows := make([]map[string]any, 0, len(parts["chunks"]))
|
||||
for _, chunk := range parts["chunks"] {
|
||||
id, _ := chunk["id"].(string)
|
||||
knowledge, _ := chunk["knowledge_id"].(string)
|
||||
content, _ := chunk["content"].(string)
|
||||
indexRows = append(indexRows, map[string]any{"id": id, "source": map[string]any{
|
||||
"tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "knowledge_id": knowledge,
|
||||
"chunk_id": id, "source_id": id, "content": content, "embedding": []float64{0.1, 0.2}}})
|
||||
}
|
||||
indexPart, err := archive.Put(bytes.NewReader(encode(indexRows)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", 1
|
||||
indexPart.Kind, indexPart.Name, indexPart.Count = "index", "chunks", int64(len(indexRows))
|
||||
manifestParts = append(manifestParts, indexPart)
|
||||
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
|
||||
Format: librarysnapshot.Format, Scope: "internal", TenantID: wsSourceTenant, KnowledgeBaseID: wsSourceKB,
|
||||
Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB,
|
||||
CreatedAt: time.Now().UTC(), Parts: manifestParts,
|
||||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4",
|
||||
|
||||
Reference in New Issue
Block a user