e0fc2c32b8
build / build (push) Successful in 2m11s
同实例第二个目标 KB 与旧库共享源 UUID 时,计划/写入/校验/恢复按目标范围派生稳定身份; 计划持久化 target_id_mode,旧计划与写前镜像不二次映射;资源保持共享 ID。
472 lines
18 KiB
Go
472 lines
18 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// 写集合台账:一次发布的"明确写集合 + 写前存在状态 + 预备份时刻整实例证据"。
|
||
//
|
||
// 为什么必须是持久记录而不是恢复时现算:
|
||
// 恢复发生在写失败之后,目标已经变了。此刻按"目标里没人引用的资源就是残渣"去删,
|
||
// 会连历史孤儿、同租户其它知识库暂未绑定的资源一起删掉——而预备份里没有这些行,
|
||
// 删了就是不可恢复的数据丢失(真实缺陷)。唯一可靠的口径是写之前就记下
|
||
// "这次要写哪些精确 ID、哪些写前就已存在",恢复只删"写前不存在 ∧ 由本次写入"的那些。
|
||
|
||
// writeSetDirName 是台账在归档根下的固定目录,与快照同级:同一块持久卷,
|
||
// 重启后按摘要直接取回。
|
||
const writeSetDirName = "writesets"
|
||
|
||
func writeSetPath(archiveRoot, digest string) string {
|
||
return filepath.Join(archiveRoot, writeSetDirName, digest+".json")
|
||
}
|
||
|
||
// PersistWriteSet 把台账按内容寻址落盘(文件名即 sha256),随后只读。
|
||
// 幂等:同摘要文件已存在且内容一致就直接返回;内容不一致说明摘要被占用,必须失败。
|
||
func PersistWriteSet(archiveRoot string, set *libraryrelease.WriteSet) (string, error) {
|
||
if set.Digest == "" {
|
||
if err := set.Seal(); err != nil {
|
||
return "", err
|
||
}
|
||
}
|
||
if len(set.Universe) == 0 {
|
||
return "", errors.New("写集合缺少整实例内容证据,拒绝落盘")
|
||
}
|
||
canonical, err := set.CanonicalBytes()
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(canonical)
|
||
digest := hex.EncodeToString(sum[:])
|
||
if digest != set.Digest {
|
||
return "", errors.New("写集合摘要与内容不符,拒绝落盘")
|
||
}
|
||
raw, err := json.Marshal(*set)
|
||
if err != nil {
|
||
return "", fmt.Errorf("编码写集合失败: %w", err)
|
||
}
|
||
path := writeSetPath(archiveRoot, digest)
|
||
if err := persistImmutableJSON(path, raw); err != nil {
|
||
return "", err
|
||
}
|
||
return path, nil
|
||
}
|
||
|
||
// LoadWriteSet 按摘要取回台账:文件名、字节摘要、内容摘要三者必须一致。
|
||
// 任何一步不符都返回错误——治理库里的引用指向的就是这个摘要。
|
||
func LoadWriteSet(archiveRoot, digest string) (*libraryrelease.WriteSet, error) {
|
||
if digest == "" {
|
||
return nil, errors.New("没有写集合摘要,无法取回记录")
|
||
}
|
||
path := writeSetPath(archiveRoot, digest)
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("写集合记录缺失(%s)", digest)
|
||
}
|
||
var set libraryrelease.WriteSet
|
||
if err := json.Unmarshal(raw, &set); err != nil {
|
||
return nil, errors.New("写集合记录不可解析")
|
||
}
|
||
if err := set.Verify(); err != nil {
|
||
return nil, err
|
||
}
|
||
if set.Digest != digest {
|
||
return nil, errors.New("写集合记录摘要不符")
|
||
}
|
||
return &set, nil
|
||
}
|
||
|
||
// PlanWriteSet 在写第一笔之前算出"这次明确要写什么",拒绝跨范围 ID 冲突,
|
||
// 并把台账落盘。返回的记录已 Seal 过;调用方负责把摘要登记到治理库。
|
||
func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan libraryrelease.WriteSetPlan) (libraryrelease.WriteSet, error) {
|
||
if plan.JobID == "" || plan.BackupSnapshotID == "" || plan.TargetSnapshotID == "" {
|
||
return libraryrelease.WriteSet{}, errors.New("写集合缺少作业/预备份/快照身份")
|
||
}
|
||
if len(plan.Universe) == 0 {
|
||
return libraryrelease.WriteSet{}, errors.New("预备份没有整实例内容证据,拒绝在没有保护证据的情况下写目标")
|
||
}
|
||
// 资源复用的"整行内容"必须按写入时的同一套范围/存储改写来比,所以先把目标
|
||
// 存储后端解析出来;解析不到就等于写不出可解析的定位符,直接拒绝。
|
||
if err := t.resolveStorageBackend(ctx); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
// 每次预检都从空集开始:同一进程里复用目标对象时,上一次作业的"可复用"
|
||
// 判断不能泄漏到这一次。
|
||
t.reusedResources = map[string]bool{}
|
||
archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, errors.New("归档不可用")
|
||
}
|
||
defer archive.Close()
|
||
manifest, err := archive.Load(plan.TargetSnapshotID)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, errors.New("要同步的快照不可读")
|
||
}
|
||
if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
// 新计划必须从计划阶段就使用目标隔离身份:后面的归属检查、索引计划、
|
||
// 写集合登记与 Apply/Verify 都要看到同一套 ID。
|
||
if err := t.enableTargetMapping(archive, manifest); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
t.targetIDMode = libraryrelease.TargetIDModeScoped
|
||
if err := validateArchivedLibraryClosure(archive, manifest); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if err := verifyLibraryRelationalClosure(ctx, t.pool, PostgresIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID}, true); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
schemaDigest, err := t.preflightMigrationCatalog(ctx, archive, manifest)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
backupResources, err := archivedSnapshotResourceIDs(archive, plan.BackupSnapshotID)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
resourceIDs := make([]string, 0, len(backupResources))
|
||
for id := range backupResources {
|
||
resourceIDs = append(resourceIDs, id)
|
||
}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind == "resource" {
|
||
resourceIDs = append(resourceIDs, part.Name)
|
||
}
|
||
}
|
||
if err := verifyLibraryResourceOwners(ctx, t.pool, resourceIDs); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
set := libraryrelease.WriteSet{SchemaDigest: schemaDigest, Version: libraryrelease.MigrationWriteSetVersion, ContractVersion: migrationIdentityContractVersion,
|
||
SourceTenantID: manifest.TenantID, SourceKnowledgeBaseID: manifest.KnowledgeBaseID, JobID: plan.JobID, Target: plan.Target,
|
||
TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID,
|
||
BackupSnapshotID: plan.BackupSnapshotID, TargetSnapshotID: plan.TargetSnapshotID,
|
||
CreatedAt: t.now().UTC(), Universe: plan.Universe, TargetIDMode: libraryrelease.TargetIDModeScoped}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "database" && part.Kind != "wiki" {
|
||
continue
|
||
}
|
||
ids, err := t.archivedRowIdentitiesFor(archive, part)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if len(ids) == 0 {
|
||
continue
|
||
}
|
||
if part.Name == "wiki_log_entries" {
|
||
continue
|
||
} // source IDs are local; target IDs are assigned below
|
||
owners, err := t.existingRowOwners(ctx, part.Name, ids)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
for _, id := range ids {
|
||
owner, exists := owners[id]
|
||
if !exists {
|
||
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id})
|
||
continue
|
||
}
|
||
if part.Name == resourceReuseTableName {
|
||
// 资源不在预备份范围,但可能正是这个目标上一次成功同步写进去、
|
||
// 又被恢复保留下来的那份内容。只有"内容/字节相同 + 来源可审计"
|
||
// 才允许复用(不重写、不清理);否则按跨范围冲突拒绝。
|
||
if err := t.assertResourceTenant(id, owner); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
row, reuseErr := t.planReusableResource(ctx, archive, manifest, part, id, owner, plan)
|
||
if reuseErr != nil {
|
||
return libraryrelease.WriteSet{}, reuseErr
|
||
}
|
||
set.Rows = append(set.Rows, row)
|
||
continue
|
||
}
|
||
if err := t.assertWriteSetOwnership(part.Name, id, owner, backupResources); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id, Existing: true, Owner: owner})
|
||
}
|
||
}
|
||
if err := t.preflightArchiveIndex(ctx, archive, manifest); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if err := t.preflightNaturalKeys(ctx, archive, manifest); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if err := t.planIndex(ctx, archive, manifest, &set); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if err := t.planObjects(ctx, archive, manifest, &set); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
set.LocalIDs, err = t.localIDRequests(archive, manifest)
|
||
if err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if err := t.allocateLocalIDs(ctx, &set); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if err := set.Seal(); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if _, err := PersistWriteSet(t.config.ArchiveRoot, &set); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
if err := t.AdoptWriteSet(set); err != nil {
|
||
return libraryrelease.WriteSet{}, err
|
||
}
|
||
return set, nil
|
||
}
|
||
|
||
// archivedSnapshotResourceIDs 读一份快照里"属于目标自己的资源身份"。
|
||
// 资源表被多个知识库共享,只有在这份预备份里的资源行才允许被覆盖写。
|
||
func archivedSnapshotResourceIDs(archive *librarysnapshot.Archive, snapshotID string) (map[string]bool, error) {
|
||
ids := map[string]bool{}
|
||
if snapshotID == "" {
|
||
return ids, nil
|
||
}
|
||
manifest, err := archive.Load(snapshotID)
|
||
if err != nil {
|
||
return nil, errors.New("预备份快照不可读")
|
||
}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "database" || part.Name != "resources" {
|
||
continue
|
||
}
|
||
values, err := archivedResourceIDs(archive, part)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for _, id := range values {
|
||
ids[id] = true
|
||
}
|
||
}
|
||
return ids, nil
|
||
}
|
||
|
||
// existingRowOwners 读出这些身份在目标里的写前归属("<tenant>/<kb>")。
|
||
// 没有 id 列的关系表按父表 knowledges 判归属。
|
||
func (t *ProductionLibraryTarget) existingRowOwners(ctx context.Context, table string, ids []string) (map[string]string, error) {
|
||
owners := map[string]string{}
|
||
if _, ok := migrationIdentityContracts[table]; !ok {
|
||
return nil, fmt.Errorf("unknown migration table %s", table)
|
||
}
|
||
if table == "knowledge_tag_relations" {
|
||
knowledgeIDs, tagIDs := make([]string, 0, len(ids)), make([]string, 0, len(ids))
|
||
for _, id := range ids {
|
||
parts := strings.Split(id, ":")
|
||
if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validSearchIdentifier(parts[1]) {
|
||
return nil, errors.New("invalid relation tuple")
|
||
}
|
||
knowledgeIDs = append(knowledgeIDs, parts[0])
|
||
tagIDs = append(tagIDs, parts[1])
|
||
}
|
||
rows, err := t.pool.Query(ctx, `
|
||
SELECT r.knowledge_id, coalesce(r.tag_id,''), coalesce(k.tenant_id,0), coalesce(k.knowledge_base_id,'')
|
||
FROM knowledge_tag_relations r LEFT JOIN knowledges k ON k.id=r.knowledge_id
|
||
WHERE (r.knowledge_id,r.tag_id) IN (SELECT * FROM unnest($1::text[],$2::text[]))`, knowledgeIDs, tagIDs)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取目标 %s 归属失败", table)
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var knowledge, tag, kb string
|
||
var tenant uint64
|
||
if err := rows.Scan(&knowledge, &tag, &tenant, &kb); err != nil {
|
||
return nil, err
|
||
}
|
||
owners[knowledge+":"+tag] = fmt.Sprintf("%d/%s", tenant, kb)
|
||
}
|
||
return owners, rows.Err()
|
||
}
|
||
statement := fmt.Sprintf(`SELECT id, tenant_id, coalesce(knowledge_base_id,'') FROM %s WHERE id = ANY($1::text[])`, table)
|
||
if table == "resource_bindings" {
|
||
// 绑定表没有 knowledge_base_id:归属只能走父表,否则"同租户"就会被当成"自己的"。
|
||
statement = `SELECT b.id, b.tenant_id,
|
||
coalesce(k.knowledge_base_id, c.knowledge_base_id, '')
|
||
FROM resource_bindings b
|
||
LEFT JOIN knowledges k ON b.owner_type='knowledge' AND k.id=b.owner_id
|
||
LEFT JOIN chunks c ON b.owner_type='chunk' AND c.id=b.owner_id
|
||
WHERE b.id = ANY($1::text[])`
|
||
} else if !tablesWithKB[table] {
|
||
statement = fmt.Sprintf(`SELECT id, tenant_id, '' FROM %s WHERE id = ANY($1::text[])`, table)
|
||
}
|
||
if table == "wiki_log_entries" {
|
||
statement = `SELECT id::text, tenant_id, coalesce(knowledge_base_id,'') FROM wiki_log_entries WHERE id = ANY($1::bigint[])`
|
||
}
|
||
rows, err := t.pool.Query(ctx, statement, ids)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取目标 %s 归属失败", table)
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var id, kb string
|
||
var tenant uint64
|
||
if err := rows.Scan(&id, &tenant, &kb); err != nil {
|
||
return nil, err
|
||
}
|
||
owners[id] = fmt.Sprintf("%d/%s", tenant, kb)
|
||
}
|
||
return owners, rows.Err()
|
||
}
|
||
|
||
// assertResourceTenant 只检查"同 ID 是否属于别的租户":resources 没有
|
||
// knowledge_base_id,同租户不等于同知识库,范围问题交给后续的复用证明。
|
||
func (t *ProductionLibraryTarget) assertResourceTenant(id, owner string) error {
|
||
if owner != fmt.Sprintf("%d/", t.scope.TenantID) {
|
||
return fmt.Errorf("资源 ID 冲突:%s 已属于租户/范围 %s,拒绝覆盖", id, owner)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// assertWriteSetOwnership 是跨范围 ID 冲突的预检:同 ID 已经是别人的行时,
|
||
// 写入会改掉别人的数据(resources 尤其危险:ON CONFLICT 会整行覆盖)。
|
||
// 宁可整次拒绝,也不许"覆盖一下看起来也能跑"。
|
||
func (t *ProductionLibraryTarget) assertWriteSetOwnership(table, id, owner string, backupResources map[string]bool) error {
|
||
target := fmt.Sprintf("%d/%s", t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||
if table == "resources" {
|
||
// 资源只有租户:同租户不等于同知识库,必须落在预备份范围内才算自己的。
|
||
if err := t.assertResourceTenant(id, owner); err != nil {
|
||
return err
|
||
}
|
||
if !backupResources[id] {
|
||
return fmt.Errorf("资源 ID 冲突:%s 存在于目标租户但不在预备份范围内(属于其它知识库),拒绝覆盖", id)
|
||
}
|
||
return nil
|
||
}
|
||
if owner != target && owner != fmt.Sprintf("%d/", t.scope.TenantID) {
|
||
return fmt.Errorf("%s ID 冲突:%s 已属于 %s,拒绝写入", table, id, owner)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// clearJobCreatedRows 只删"写前不存在 ∧ 由本次写入 ∧ 不在恢复快照里"的精确身份,
|
||
// 并且对资源额外要求它仍然没有任何绑定/正文引用(作业失败期间被别的知识库接管
|
||
// 使用的资源不能删)。
|
||
//
|
||
// 为什么要减掉"快照里的身份":这份台账可能是为另一次动作算的(例如手工从某份
|
||
// 预备份修复目标)。只要这一行就是恢复要回到的内容,它就不是残渣——按台账删掉
|
||
// 等于把刚恢复的行再删一次。
|
||
func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set libraryrelease.WriteSet,
|
||
restored map[string]map[string]bool) error {
|
||
tx, err := t.pool.Begin(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer tx.Rollback(context.WithoutCancel(ctx))
|
||
|
||
// 子表先删:resource_bindings → chunks/knowledges、wiki 子表 → wiki 页 → 资源。
|
||
for _, table := range []string{"resource_bindings", "chunks", "knowledges", "knowledge_tag_relations",
|
||
"wiki_page_issues", "wiki_log_entries", "wiki_pages", "wiki_folders", "knowledge_tags", "resources"} {
|
||
ids := make([]string, 0, 8)
|
||
for _, id := range set.CreatedIDs(table) {
|
||
if restored[table][id] {
|
||
continue
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if len(ids) == 0 {
|
||
continue
|
||
}
|
||
statement := fmt.Sprintf(`DELETE FROM %s WHERE (%s)::text = ANY($1::text[])`, table, identityExpr(table))
|
||
if table == "resources" {
|
||
// Lock parent keys through grant check and delete; a concurrent FK insert
|
||
// must wait rather than race the protection query and be cascaded away.
|
||
locked, err := tx.Query(ctx, `SELECT id FROM resources WHERE id=ANY($1::text[]) FOR UPDATE`, ids)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for locked.Next() {
|
||
var id string
|
||
if err = locked.Scan(&id); err != nil {
|
||
locked.Close()
|
||
return err
|
||
}
|
||
}
|
||
locked.Close()
|
||
if err = locked.Err(); err != nil {
|
||
return err
|
||
}
|
||
if err = ensureNoResourceGrants(ctx, tx, ids); err != nil {
|
||
return err
|
||
}
|
||
|
||
if err := ensureCreatedResourcesUnreferenced(ctx, tx, ids); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if _, err := tx.Exec(ctx, statement, ids); err != nil {
|
||
return fmt.Errorf("清理本次新增的 %s 失败: %w", table, err)
|
||
}
|
||
}
|
||
// 复核:本次新建的行一个都不许剩。资源若因"被引用"而保留,那说明恢复没回到
|
||
// 预备份状态,必须报出来让维护保持,而不是静默放过。
|
||
leftovers := make([]string, 0, 4)
|
||
for _, table := range productionLibraryTables {
|
||
ids := make([]string, 0, 8)
|
||
for _, id := range set.CreatedIDs(table) {
|
||
if restored[table][id] {
|
||
continue
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if len(ids) == 0 {
|
||
continue
|
||
}
|
||
statement := fmt.Sprintf(`SELECT (%s)::text FROM %s WHERE (%s)::text = ANY($1::text[])`,
|
||
identityExpr(table), table, identityExpr(table))
|
||
rows, err := tx.Query(ctx, statement, ids)
|
||
if err != nil {
|
||
return fmt.Errorf("复核本次新增的 %s 失败: %w", table, err)
|
||
}
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
rows.Close()
|
||
return err
|
||
}
|
||
leftovers = append(leftovers, table+"/"+id)
|
||
}
|
||
rows.Close()
|
||
if err := rows.Err(); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if len(leftovers) > 0 {
|
||
sort.Strings(leftovers)
|
||
return fmt.Errorf("恢复后仍留下本次写入的 %d 行(%v):目标未回到预备份状态,不能当作已恢复",
|
||
len(leftovers), leftovers)
|
||
}
|
||
return tx.Commit(ctx)
|
||
}
|
||
|
||
// snapshotRowIdentities 读出"恢复要用这份快照"的全部行身份(按表)。
|
||
func (t *ProductionLibraryTarget) snapshotRowIdentities(archive *librarysnapshot.Archive,
|
||
manifest librarysnapshot.Manifest) (map[string]map[string]bool, error) {
|
||
identities := map[string]map[string]bool{}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "database" && part.Kind != "wiki" {
|
||
continue
|
||
}
|
||
ids, err := archivedRowIdentities(archive, part)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
identities[part.Name] = map[string]bool{}
|
||
for _, id := range ids {
|
||
identities[part.Name][id] = true
|
||
}
|
||
}
|
||
return identities, nil
|
||
}
|