姚章浩
5defe238e0
build / build (push) Successful in 1m34s
真实事故：源 KB(10000/0c84a098) 同步到目标 KB(10001/e64e82b5) 时，归档行被原样 插入目标库，落在来源租户/知识库；资源阶段随即报"生产缺少资源 ... 的存储路径" （资源查询按目标租户过滤，看不见刚写进去的行）。回滚只按目标范围回放，于是页面 报"已恢复到预备份"，目标却多出 4 文件/38 分块/21 资源/4 绑定。 改： - 写入与校验共用 retargetRow：tenant_id / knowledge_base_id / 资源定位符(后端、 provider、桶、租户前缀) / storage_backend_id / provider / embedding_model_id 统一映射到目标范围；行里的来源身份对不上就失败，宁可不写。 - Verify 增加"本次写集合仍以来源身份残留"判定；Restore 增加整库指纹比对（预备份 时刻 vs 恢复后）与孤儿资源清理，回滚后用 Verify 复核，恢复工具退出 0 不再等于 恢复完成。 - 新增 cmd/libraryresidue：按归档身份精确、可备份、可复核地清理已知的范围外残留。 验证：make verify 全绿；真实 PG+ES 用例复现"源目标 KB 不同 + 目标非空 + 资源阶段 失败"，断言回滚后无来源范围残留、其它租户行未动、恢复后内容=预备份；正向用例断言 行落在目标范围且资源定位符指向目标存储。
202 lines
7.8 KiB
Go
202 lines
7.8 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// ProductionBackupStore 在生产写入前创建**独立完整预备份**：从生产读一次整库
|
||
// （PG RepeatableRead + ES 导出一致快照 + 资源对象），密封成归档里的一个版本。
|
||
//
|
||
// 为什么要有静默检查：预备份期间生产若仍在变，导出的 PG/ES/对象就不是同一个切点，
|
||
// 恢复它等于恢复到"没人见过的状态"。所以导出前后各读一次行数，不一致就失败，
|
||
// 让作业重试，而不是拿一个不一致的备份当恢复点。
|
||
type ProductionBackupStore struct {
|
||
pool *pgxpool.Pool
|
||
search *SearchExporter
|
||
objects ObjectReader
|
||
archiveRoot string
|
||
scope string
|
||
identity PostgresIdentity
|
||
compatibility librarysnapshot.Compatibility
|
||
maxDocuments int
|
||
maxObjectByte int64
|
||
now func() time.Time
|
||
}
|
||
|
||
type ProductionBackupConfig struct {
|
||
Pool *pgxpool.Pool
|
||
Search *SearchExporter
|
||
Objects ObjectReader
|
||
ArchiveRoot string
|
||
Scope string
|
||
Identity PostgresIdentity
|
||
Compatibility librarysnapshot.Compatibility
|
||
MaxDocuments int
|
||
MaxObjectByte int64
|
||
Now func() time.Time
|
||
}
|
||
|
||
func NewProductionBackupStore(config ProductionBackupConfig) (*ProductionBackupStore, error) {
|
||
if config.Pool == nil || config.Search == nil || config.Objects == nil || config.ArchiveRoot == "" ||
|
||
(config.Scope != "internal" && config.Scope != "external") || config.Identity.TenantID == 0 ||
|
||
!validSearchIdentifier(config.Identity.KnowledgeBaseID) || config.MaxDocuments < 1 {
|
||
return nil, errors.New("生产预备份配置不完整")
|
||
}
|
||
if config.MaxObjectByte <= 0 {
|
||
config.MaxObjectByte = 256 * 1024 * 1024
|
||
}
|
||
now := config.Now
|
||
if now == nil {
|
||
now = time.Now
|
||
}
|
||
return &ProductionBackupStore{pool: config.Pool, search: config.Search, objects: config.Objects,
|
||
archiveRoot: config.ArchiveRoot, scope: config.Scope, identity: config.Identity,
|
||
compatibility: config.Compatibility, maxDocuments: config.MaxDocuments,
|
||
maxObjectByte: config.MaxObjectByte, now: now}, nil
|
||
}
|
||
|
||
// tableFingerprint 是一张表的"数量 + 内容摘要"指纹：同数量改写也会被识别。
|
||
type tableFingerprint struct {
|
||
Count int64
|
||
Digest string
|
||
}
|
||
|
||
// fingerprints 读取当前生产指纹；Prepare 在导出前后各取一次做一致性判断。
|
||
func (s *ProductionBackupStore) fingerprints(ctx context.Context) (map[string]tableFingerprint, error) {
|
||
result := map[string]tableFingerprint{}
|
||
for _, table := range productionLibraryTables {
|
||
rows, err := s.pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`,
|
||
table, table, scopeWhere(table)), s.identity.TenantID, s.identity.KnowledgeBaseID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取生产 %s 指纹失败", table)
|
||
}
|
||
digests := make([]string, 0, 64)
|
||
for rows.Next() {
|
||
var raw map[string]json.RawMessage
|
||
if err := rows.Scan(&raw); err != nil {
|
||
rows.Close()
|
||
return nil, err
|
||
}
|
||
digest, err := canonicalRowDigest(raw)
|
||
if err != nil {
|
||
rows.Close()
|
||
return nil, err
|
||
}
|
||
digests = append(digests, digest)
|
||
}
|
||
rows.Close()
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
result[table] = tableFingerprint{Count: int64(len(digests)), Digest: combineRowDigests(digests)}
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
|
||
archive, err := librarysnapshot.Open(s.archiveRoot, 256*1024*1024)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, errors.New("归档不可用，无法创建预备份")
|
||
}
|
||
defer archive.Close()
|
||
createdAt := s.now().UTC()
|
||
before, err := s.fingerprints(ctx)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, err
|
||
}
|
||
// 整实例指纹:恢复之后要拿它证明"多写的、少写的都没有"。只按目标范围比,
|
||
// 写到范围外的行会被漏掉(真实事故:恢复报成功但目标多 67 行)。
|
||
universeBefore, err := universeFingerprint(ctx, s.pool)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, err
|
||
}
|
||
pg, err := ExportLibrary(ctx, s.pool, s.identity)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("读取生产整库失败: %w", err)
|
||
}
|
||
// 整库导出用 ExportLibrary（按租户+知识库，不带单文档身份），
|
||
// 与封存路径复用同一份实现，避免两套范围规则。
|
||
es, err := s.search.ExportLibrary(ctx, SearchIdentity{TenantID: s.identity.TenantID,
|
||
KnowledgeBaseID: s.identity.KnowledgeBaseID}, s.maxDocuments)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("读取生产索引失败: %w", err)
|
||
}
|
||
if es.MappingSHA256 == "" {
|
||
// 生产索引不存在：只有数据库也为空时才是合法的"空基线"，否则说明索引丢了。
|
||
if len(pg.Table("knowledges").Rows) != 0 || len(pg.Table("chunks").Rows) != 0 {
|
||
return libraryrelease.BackupRef{}, errors.New("生产索引缺失但数据库非空：先修索引再发布")
|
||
}
|
||
} else if es.MappingSHA256 != s.compatibility.IndexMappingSHA256 {
|
||
// 契约不符说明生产换了索引映射：此时备份出来的东西无法用同一套恢复。
|
||
// 唯一例外是"空生产"：零文档零分块时映射差异不影响可恢复性。
|
||
emptyProduction := len(pg.Table("knowledges").Rows) == 0 && len(pg.Table("chunks").Rows) == 0 && len(es.Documents) == 0
|
||
if !emptyProduction {
|
||
return libraryrelease.BackupRef{}, errors.New("生产索引映射与归档契约不一致")
|
||
}
|
||
}
|
||
// ArchiveLibraryCandidate 自己按 resources 行读取对象字节并计算摘要，
|
||
// 所以这里直接给它生产对象读取器，不再重复导出一次资源。
|
||
snapshotID, err := ArchiveLibraryCandidate(ctx, archive, s.scope, s.identity, s.compatibility, pg, es, s.objects, createdAt)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("密封预备份失败: %w", err)
|
||
}
|
||
// 静默检查：导出期间生产内容不能变。只比行数会被"同数量改写"骗过，
|
||
// 所以这里比对逐行内容摘要（行数 + 身份 + 内容）。
|
||
after, err := s.fingerprints(ctx)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, err
|
||
}
|
||
for table, beforeState := range before {
|
||
if after[table] != beforeState {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间生产 %s 内容发生变化，切点不一致", table)
|
||
}
|
||
}
|
||
universeAfter, err := universeFingerprint(ctx, s.pool)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, err
|
||
}
|
||
if diff := universeDiff(universeBefore, universeAfter); diff != "" {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间整实例知识域发生变化（%s），切点不一致", diff)
|
||
}
|
||
for table, want := range current.Counts {
|
||
if table == "index" {
|
||
continue
|
||
}
|
||
if after[table].Count != want {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间生产 %s 行数从 %d 变为 %d", table, want, after[table].Count)
|
||
}
|
||
}
|
||
manifest, err := archive.Load(snapshotID)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, errors.New("预备份封存后无法校验")
|
||
}
|
||
var total int64
|
||
for _, part := range manifest.Parts {
|
||
total += part.Bytes
|
||
}
|
||
return libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID, Bytes: total,
|
||
CreatedAt: createdAt, Universe: universeBefore}, nil
|
||
}
|
||
|
||
func (s *ProductionBackupStore) rowCounts(ctx context.Context) (map[string]int64, error) {
|
||
counts := map[string]int64{}
|
||
for _, table := range productionLibraryTables {
|
||
var count int64
|
||
if err := s.pool.QueryRow(ctx, fmt.Sprintf(`SELECT count(*) FROM %s WHERE %s`,
|
||
table, scopeWhere(table)), s.identity.TenantID, s.identity.KnowledgeBaseID).Scan(&count); err != nil {
|
||
return nil, fmt.Errorf("读取生产 %s 行数失败", table)
|
||
}
|
||
counts[table] = count
|
||
}
|
||
return counts, nil
|
||
}
|