0252492e53
同步 A 成功后手工恢复 B,resources 表既不删行也不删对象;再同步 A 时, resources 里那 21 行以“同租户但不在预备份范围”被判成别人的知识库,整次 同步在写第一笔之前被拒。 “目标里没人引用”不是归属证据:历史孤儿、其它知识库暂未绑定的资源都满足 它。改成唯一放行路径——可审计复用(v071/library_reuse.go): 1. 持久来源:这个目标成功同步过且登记过生效版本的快照,或它自己的预备份, 且同一资源行在来源里内容一致; 2. 整行内容(除易变时间戳)与本次写入将要落地的行完全一致; 3. 该资源的对象字节与本次快照归档摘要一致。 三条同时成立才允许复用:写入阶段跳过它(不重写行、不重传对象),写集合里 记 existing+reused,恢复不清理;证据单独落盘 reuse/<digest>.json,摘要写进 写集合行,不冒充历史台账。内容不同、来源范围不同、或没有持久来源一律继续 拒绝,且拒绝发生在写入之前。 为什么把预备份也算来源:恢复作业的写集合同样会把“本目标历史状态里已有的 资源”判成跨范围冲突,而它们本来就出自这个目标自己的快照。 验证:隔离 PG+ES+MinIO 真服务下新增 5 个场景用例(复用成功且 xmin 不变、 无来源拒绝、内容不同拒绝、同内容不同来源拒绝、wiki 半写回滚只清本次新建); v071 整包与全仓 go test 通过;make verify 15 项全绿。
414 lines
16 KiB
Go
414 lines
16 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// 写集合台账:一次发布的"明确写集合 + 写前存在状态 + 预备份时刻整实例证据"。
|
||
//
|
||
// 为什么必须是持久记录而不是恢复时现算:
|
||
// 恢复发生在写失败之后,目标已经变了。此刻按"目标里没人引用的资源就是残渣"去删,
|
||
// 会连历史孤儿、同租户其它知识库暂未绑定的资源一起删掉——而预备份里没有这些行,
|
||
// 删了就是不可恢复的数据丢失(真实缺陷)。唯一可靠的口径是写之前就记下
|
||
// "这次要写哪些精确 ID、哪些写前就已存在",恢复只删"写前不存在 ∧ 由本次写入"的那些。
|
||
|
||
// writeSetDirName 是台账在归档根下的固定目录,与快照同级:同一块持久卷,
|
||
// 重启后按摘要直接取回。
|
||
const writeSetDirName = "writesets"
|
||
|
||
func writeSetPath(archiveRoot, digest string) string {
|
||
return filepath.Join(archiveRoot, writeSetDirName, digest+".json")
|
||
}
|
||
|
||
// PersistWriteSet 把台账按内容寻址落盘(文件名即 sha256),随后只读。
|
||
// 幂等:同摘要文件已存在且内容一致就直接返回;内容不一致说明摘要被占用,必须失败。
|
||
func PersistWriteSet(archiveRoot string, set *libraryrelease.WriteSet) (string, error) {
|
||
if set.Digest == "" {
|
||
if err := set.Seal(); err != nil {
|
||
return "", err
|
||
}
|
||
}
|
||
if len(set.Universe) == 0 {
|
||
return "", errors.New("写集合缺少整实例内容证据,拒绝落盘")
|
||
}
|
||
canonical, err := set.CanonicalBytes()
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(canonical)
|
||
digest := hex.EncodeToString(sum[:])
|
||
if digest != set.Digest {
|
||
return "", errors.New("写集合摘要与内容不符,拒绝落盘")
|
||
}
|
||
raw, err := json.Marshal(*set)
|
||
if err != nil {
|
||
return "", fmt.Errorf("编码写集合失败: %w", err)
|
||
}
|
||
directory := filepath.Join(archiveRoot, writeSetDirName)
|
||
if err := os.MkdirAll(directory, 0700); err != nil {
|
||
return "", fmt.Errorf("创建写集合目录失败: %w", err)
|
||
}
|
||
path := writeSetPath(archiveRoot, digest)
|
||
if existing, readErr := os.ReadFile(path); readErr == nil {
|
||
var recorded libraryrelease.WriteSet
|
||
if json.Unmarshal(existing, &recorded) != nil || recorded.Verify() != nil || recorded.Digest != digest {
|
||
return "", errors.New("写集合记录同名但内容不符,拒绝覆盖")
|
||
}
|
||
return path, nil
|
||
} else if !errors.Is(readErr, os.ErrNotExist) {
|
||
return "", fmt.Errorf("读取已有写集合失败: %w", readErr)
|
||
}
|
||
temporary := path + ".tmp"
|
||
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||
if err != nil {
|
||
return "", fmt.Errorf("写入写集合失败: %w", err)
|
||
}
|
||
if _, err := file.Write(raw); err != nil {
|
||
file.Close()
|
||
os.Remove(temporary)
|
||
return "", fmt.Errorf("写入写集合失败: %w", err)
|
||
}
|
||
if err := file.Sync(); err != nil {
|
||
file.Close()
|
||
os.Remove(temporary)
|
||
return "", fmt.Errorf("写集合未落盘: %w", err)
|
||
}
|
||
if err := file.Close(); err != nil {
|
||
os.Remove(temporary)
|
||
return "", fmt.Errorf("写集合未落盘: %w", err)
|
||
}
|
||
if err := os.Chmod(temporary, 0444); err != nil {
|
||
os.Remove(temporary)
|
||
return "", fmt.Errorf("写集合只读化失败: %w", err)
|
||
}
|
||
if err := os.Rename(temporary, path); err != nil {
|
||
os.Remove(temporary)
|
||
return "", fmt.Errorf("写集合落盘失败: %w", err)
|
||
}
|
||
return path, nil
|
||
}
|
||
|
||
// LoadWriteSet 按摘要取回台账:文件名、字节摘要、内容摘要三者必须一致。
|
||
// 任何一步不符都返回错误——治理库里的引用指向的就是这个摘要。
|
||
func LoadWriteSet(archiveRoot, digest string) (*libraryrelease.WriteSet, error) {
|
||
if digest == "" {
|
||
return nil, errors.New("没有写集合摘要,无法取回记录")
|
||
}
|
||
path := writeSetPath(archiveRoot, digest)
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("写集合记录缺失(%s)", digest)
|
||
}
|
||
var set libraryrelease.WriteSet
|
||
if err := json.Unmarshal(raw, &set); err != nil {
|
||
return nil, errors.New("写集合记录不可解析")
|
||
}
|
||
if err := set.Verify(); err != nil {
|
||
return nil, err
|
||
}
|
||
if set.Digest != digest {
|
||
return nil, errors.New("写集合记录摘要不符")
|
||
}
|
||
return &set, nil
|
||
}
|
||
|
||
// PlanWriteSet 在写第一笔之前算出"这次明确要写什么",拒绝跨范围 ID 冲突,
|
||
// 并把台账落盘。返回的记录已 Seal 过;调用方负责把摘要登记到治理库。
|
||
func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
|
||
if plan.JobID == "" || plan.BackupSnapshotID == "" || plan.TargetSnapshotID == "" {
|
||
return libraryrelease.WriteSet{}, errors.New("写集合缺少作业/预备份/快照身份")
|
||
}
|
||
if len(plan.Universe) == 0 {
|
||
return libraryrelease.WriteSet{}, errors.New("预备份没有整实例内容证据,拒绝在没有保护证据的情况下写目标")
|
||
}
|
||
// 资源复用的"整行内容"必须按写入时的同一套范围/存储改写来比,所以先把目标
|
||
// 存储后端解析出来;解析不到就等于写不出可解析的定位符,直接拒绝。
|
||
if err := t.resolveStorageBackend(ctx); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
// 每次预检都从空集开始:同一进程里复用目标对象时,上一次作业的"可复用"
|
||
// 判断不能泄漏到这一次。
|
||
t.reusedResources = map[string]bool{}
|
||
archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, errors.New("归档不可用")
|
||
}
|
||
defer archive.Close()
|
||
manifest, err := archive.Load(plan.TargetSnapshotID)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, errors.New("要同步的快照不可读")
|
||
}
|
||
if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
backupResources, err := archivedSnapshotResourceIDs(archive, plan.BackupSnapshotID)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: plan.JobID, Target: plan.Target,
|
||
TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID,
|
||
BackupSnapshotID: plan.BackupSnapshotID, TargetSnapshotID: plan.TargetSnapshotID,
|
||
CreatedAt: t.now().UTC(), Universe: plan.Universe}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "database" && part.Kind != "wiki" {
|
||
continue
|
||
}
|
||
ids, err := archivedRowIdentities(archive, part)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if len(ids) == 0 {
|
||
continue
|
||
}
|
||
owners, err := t.existingRowOwners(ctx, part.Name, ids)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
for _, id := range ids {
|
||
owner, exists := owners[id]
|
||
if !exists {
|
||
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id})
|
||
continue
|
||
}
|
||
if part.Name == resourceReuseTableName && !backupResources[id] {
|
||
// 资源不在预备份范围,但可能正是这个目标上一次成功同步写进去、
|
||
// 又被恢复保留下来的那份内容。只有"内容/字节相同 + 来源可审计"
|
||
// 才允许复用(不重写、不清理);否则按跨范围冲突拒绝。
|
||
if err := t.assertResourceTenant(id, owner); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
row, reuseErr := t.planReusableResource(ctx, archive, manifest, part, id, owner, plan)
|
||
if reuseErr != nil {
|
||
return libraryrelease.WriteSet{}, reuseErr
|
||
}
|
||
set.Rows = append(set.Rows, row)
|
||
continue
|
||
}
|
||
if err := t.assertWriteSetOwnership(part.Name, id, owner, backupResources); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id, Existing: true, Owner: owner})
|
||
}
|
||
}
|
||
if err := set.Seal(); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if _, err := PersistWriteSet(t.config.ArchiveRoot, &set); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
return set, nil
|
||
}
|
||
|
||
// archivedSnapshotResourceIDs 读一份快照里"属于目标自己的资源身份"。
|
||
// 资源表被多个知识库共享,只有在这份预备份里的资源行才允许被覆盖写。
|
||
func archivedSnapshotResourceIDs(archive *librarysnapshot.Archive, snapshotID string) (map[string]bool, error) {
|
||
ids := map[string]bool{}
|
||
if snapshotID == "" {
|
||
return ids, nil
|
||
}
|
||
manifest, err := archive.Load(snapshotID)
|
||
if err != nil {
|
||
return nil, errors.New("预备份快照不可读")
|
||
}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "database" || part.Name != "resources" {
|
||
continue
|
||
}
|
||
values, err := archivedResourceIDs(archive, part)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for _, id := range values {
|
||
ids[id] = true
|
||
}
|
||
}
|
||
return ids, nil
|
||
}
|
||
|
||
// existingRowOwners 读出这些身份在目标里的写前归属("<tenant>/<kb>")。
|
||
// 没有 id 列的关系表按父表 knowledges 判归属。
|
||
func (t *ProductionLibraryTarget) existingRowOwners(ctx context.Context, table string, ids []string) (map[string]string, error) {
|
||
owners := map[string]string{}
|
||
if table == "knowledge_tag_relations" {
|
||
rows, err := t.pool.Query(ctx, `
|
||
SELECT r.knowledge_id, coalesce(r.tag_id,''), coalesce(k.tenant_id,0), coalesce(k.knowledge_base_id,'')
|
||
FROM knowledge_tag_relations r LEFT JOIN knowledges k ON k.id=r.knowledge_id
|
||
WHERE r.knowledge_id = ANY($1::text[])`, ids)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取目标 %s 归属失败", table)
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var knowledge, tag, kb string
|
||
var tenant uint64
|
||
if err := rows.Scan(&knowledge, &tag, &tenant, &kb); err != nil {
|
||
return nil, err
|
||
}
|
||
owners[knowledge+":"+tag] = fmt.Sprintf("%d/%s", tenant, kb)
|
||
}
|
||
return owners, rows.Err()
|
||
}
|
||
statement := fmt.Sprintf(`SELECT id, tenant_id, coalesce(knowledge_base_id,'') FROM %s WHERE id = ANY($1::text[])`, table)
|
||
if table == "resource_bindings" {
|
||
// 绑定表没有 knowledge_base_id:归属只能走父表,否则"同租户"就会被当成"自己的"。
|
||
statement = `SELECT b.id, b.tenant_id,
|
||
coalesce(k.knowledge_base_id, c.knowledge_base_id, '')
|
||
FROM resource_bindings b
|
||
LEFT JOIN knowledges k ON b.owner_type='knowledge' AND k.id=b.owner_id
|
||
LEFT JOIN chunks c ON b.owner_type='chunk' AND c.id=b.owner_id
|
||
WHERE b.id = ANY($1::text[])`
|
||
} else if !tablesWithKB[table] {
|
||
statement = fmt.Sprintf(`SELECT id, tenant_id, '' FROM %s WHERE id = ANY($1::text[])`, table)
|
||
}
|
||
rows, err := t.pool.Query(ctx, statement, ids)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取目标 %s 归属失败", table)
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var id, kb string
|
||
var tenant uint64
|
||
if err := rows.Scan(&id, &tenant, &kb); err != nil {
|
||
return nil, err
|
||
}
|
||
owners[id] = fmt.Sprintf("%d/%s", tenant, kb)
|
||
}
|
||
return owners, rows.Err()
|
||
}
|
||
|
||
// assertResourceTenant 只检查"同 ID 是否属于别的租户":resources 没有
|
||
// knowledge_base_id,同租户不等于同知识库,范围问题交给后续的复用证明。
|
||
func (t *ProductionLibraryTarget) assertResourceTenant(id, owner string) error {
|
||
if owner != fmt.Sprintf("%d/", t.scope.TenantID) {
|
||
return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// assertWriteSetOwnership 是跨范围 ID 冲突的预检:同 ID 已经是别人的行时,
|
||
// 写入会改掉别人的数据(resources 尤其危险:ON CONFLICT 会整行覆盖)。
|
||
// 宁可整次拒绝,也不许"覆盖一下看起来也能跑"。
|
||
func (t *ProductionLibraryTarget) assertWriteSetOwnership(table, id, owner string, backupResources map[string]bool) error {
|
||
target := fmt.Sprintf("%d/%s", t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||
if table == "resources" {
|
||
// 资源只有租户:同租户不等于同知识库,必须落在预备份范围内才算自己的。
|
||
if err := t.assertResourceTenant(id, owner); err != nil {
|
||
return err
|
||
}
|
||
if !backupResources[id] {
|
||
return fmt.Errorf("资源 ID 冲突:%s 存在于目标租户但不在预备份范围内(属于其它知识库),拒绝覆盖", id)
|
||
}
|
||
return nil
|
||
}
|
||
if owner != target && owner != fmt.Sprintf("%d/", t.scope.TenantID) {
|
||
return fmt.Errorf("%s ID 冲突:%s 已属于 %s,拒绝写入", table, id, owner)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// clearJobCreatedRows 只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复快照里"的精确身份,
|
||
// 并且对资源额外要求它仍然没有任何绑定/正文引用(作业失败期间被别的知识库接管
|
||
// 使用的资源不能删)。
|
||
//
|
||
// 为什么要减掉"快照里的身份":这份台账可能是为另一次动作算的(例如手工从某份
|
||
// 预备份修复目标)。只要这一行就是恢复要回到的内容,它就不是残渣——按台账删掉
|
||
// 等于把刚恢复的行再删一次。
|
||
func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set libraryrelease.WriteSet,
|
||
restored map[string]map[string]bool) error {
|
||
// 子表先删:resource_bindings → chunks/knowledges、wiki 子表 → wiki 页 → 资源。
|
||
for _, table := range []string{"resource_bindings", "chunks", "knowledges", "knowledge_tag_relations",
|
||
"wiki_page_issues", "wiki_log_entries", "wiki_pages", "wiki_folders", "knowledge_tags", "resources"} {
|
||
ids := make([]string, 0, 8)
|
||
for _, id := range set.CreatedIDs(table) {
|
||
if restored[table][id] {
|
||
continue
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if len(ids) == 0 {
|
||
continue
|
||
}
|
||
statement := fmt.Sprintf(`DELETE FROM %s WHERE (%s)::text = ANY($1::text[])`, table, identityExpr(table))
|
||
if table == "resources" {
|
||
statement += `
|
||
AND NOT EXISTS (SELECT 1 FROM resource_bindings b WHERE b.resource_id=resources.id)
|
||
AND NOT EXISTS (SELECT 1 FROM chunks c WHERE c.deleted_at IS NULL
|
||
AND coalesce(resources.handle,'') <> ''
|
||
AND position('resource://'||resources.handle IN coalesce(c.content,'')) > 0)`
|
||
}
|
||
if _, err := t.pool.Exec(ctx, statement, ids); err != nil {
|
||
return fmt.Errorf("清理本次新增的 %s 失败: %w", table, err)
|
||
}
|
||
}
|
||
// 复核:本次新建的行一个都不许剩。资源若因"被引用"而保留,那说明恢复没回到
|
||
// 预备份状态,必须报出来让维护保持,而不是静默放过。
|
||
leftovers := make([]string, 0, 4)
|
||
for _, table := range productionLibraryTables {
|
||
ids := make([]string, 0, 8)
|
||
for _, id := range set.CreatedIDs(table) {
|
||
if restored[table][id] {
|
||
continue
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if len(ids) == 0 {
|
||
continue
|
||
}
|
||
statement := fmt.Sprintf(`SELECT (%s)::text FROM %s WHERE (%s)::text = ANY($1::text[])`,
|
||
identityExpr(table), table, identityExpr(table))
|
||
rows, err := t.pool.Query(ctx, statement, ids)
|
||
if err != nil {
|
||
return fmt.Errorf("复核本次新增的 %s 失败: %w", table, err)
|
||
}
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
rows.Close()
|
||
return err
|
||
}
|
||
leftovers = append(leftovers, table+"/"+id)
|
||
}
|
||
rows.Close()
|
||
if err := rows.Err(); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if len(leftovers) > 0 {
|
||
sort.Strings(leftovers)
|
||
return fmt.Errorf("恢复后仍留下本次写入的 %d 行(%v):目标未回到预备份状态,不能当作已恢复",
|
||
len(leftovers), leftovers)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// snapshotRowIdentities 读出"恢复要用这份快照"的全部行身份(按表)。
|
||
func (t *ProductionLibraryTarget) snapshotRowIdentities(archive *librarysnapshot.Archive,
|
||
manifest librarysnapshot.Manifest) (map[string]map[string]bool, error) {
|
||
identities := map[string]map[string]bool{}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "database" && part.Kind != "wiki" {
|
||
continue
|
||
}
|
||
ids, err := archivedRowIdentities(archive, part)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
identities[part.Name] = map[string]bool{}
|
||
for _, id := range ids {
|
||
identities[part.Name][id] = true
|
||
}
|
||
}
|
||
return identities, nil
|
||
}
|