Files
knowledge-governance/internal/releaseadapter/v071/library_backup_store.go
T
姚章浩 1a0f99e346
build / build (push) Successful in 1m44s
fix(release): 修复发布管理F01-F11资源闭包与状态呈现
F01:捕获、写前镜像、指纹和恢复后内容校验共用按目标精确绑定的资源闭包:
从已限定范围的 resource_bindings 与正文 handle 一次提取,再按索引取资源,
不再对每个租户资源扫描全部正文表。无绑定正文引用仍保留,缺失资源仍失败。

F02/F03:恢复验证使用 capture/verify/canary/restore/finalize 自己的阶段,登记
证据不再叫新版本生效;顶部优先在飞作业;已验证且 API reason 为空时不再同屏
显示未读到状态,报告可展开。

F04-F11:显示实际目标实例/项目/库;运行中主按钮查看当前任务并禁止重复启动;
快照历史默认已登记、可筛选分页并定位所选;无真实下次计划时显示未读取;各
scope 独立 overview;新操作清旧通知;存储行按文档/分块/资源/绑定及 Wiki
页面/目录/日志分项。高影响恢复验证/恢复增加一次影响确认。

验证:隔离 PG 20k 无关资源夹具旧扫描 683ms→新导出 41ms,资源 ID 闭包相等;
Go/前端全量及 make verify 17 项通过。未部署、未做 8088 UI 验收。
2026-11-05 12:50:11 +08:00

280 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
)
// ProductionBackupStore 在生产写入前创建**独立完整预备份**:从生产读一次整库
// (PG RepeatableRead + ES 导出一致快照 + 资源对象),密封成归档里的一个版本。
//
// 为什么要有静默检查:预备份期间生产若仍在变,导出的 PG/ES/对象就不是同一个切点,
// 恢复它等于恢复到"没人见过的状态"。PG逐行指纹与对象精确版本在导出前后
// 必须一致;执行器另行核对索引before-image。此检测不能代替独占写审核。
type ProductionBackupStore struct {
pool *pgxpool.Pool
search *SearchExporter
objects ObjectReader
archiveRoot string
scope string
identity PostgresIdentity
compatibility librarysnapshot.Compatibility
maxDocuments int
maxObjectByte int64
now func() time.Time
}
type ProductionBackupConfig struct {
Pool *pgxpool.Pool
Search *SearchExporter
Objects ObjectReader
ArchiveRoot string
Scope string
Identity PostgresIdentity
Compatibility librarysnapshot.Compatibility
MaxDocuments int
MaxObjectByte int64
Now func() time.Time
}
func NewProductionBackupStore(config ProductionBackupConfig) (*ProductionBackupStore, error) {
if config.Pool == nil || config.Search == nil || config.Objects == nil || config.ArchiveRoot == "" ||
(config.Scope != "internal" && config.Scope != "external") || config.Identity.TenantID == 0 ||
!validSearchIdentifier(config.Identity.KnowledgeBaseID) || config.MaxDocuments < 1 {
return nil, errors.New("生产预备份配置不完整")
}
if config.MaxObjectByte <= 0 {
config.MaxObjectByte = 256 * 1024 * 1024
}
now := config.Now
if now == nil {
now = time.Now
}
return &ProductionBackupStore{pool: config.Pool, search: config.Search, objects: config.Objects,
archiveRoot: config.ArchiveRoot, scope: config.Scope, identity: config.Identity,
compatibility: config.Compatibility, maxDocuments: config.MaxDocuments,
maxObjectByte: config.MaxObjectByte, now: now}, nil
}
// tableFingerprint 是一张表的"数量 + 内容摘要"指纹:同数量改写也会被识别。
type tableFingerprint struct {
Count int64
Digest string
}
// fingerprints 读取当前生产指纹;Prepare 在导出前后各取一次做一致性判断。
func (s *ProductionBackupStore) fingerprints(ctx context.Context) (map[string]tableFingerprint, error) {
result := map[string]tableFingerprint{}
exporter := PostgresExporter{db: s.pool}
var bodies, bindings []json.RawMessage
for _, table := range productionLibraryTables {
if table == "resources" {
continue
}
rows, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`,
table, table, scopeWhere(table)), s.identity.TenantID, s.identity.KnowledgeBaseID)
if err != nil {
return nil, fmt.Errorf("读取生产 %s 指纹失败", table)
}
if table != "resource_bindings" && table != "knowledge_tag_relations" {
bodies = append(bodies, rows...)
}
if table == "resource_bindings" {
bindings = rows
}
digests := make([]string, 0, len(rows))
for _, raw := range rows {
var value map[string]json.RawMessage
if err := json.Unmarshal(raw, &value); err != nil {
return nil, err
}
digest, err := canonicalRowDigest(value)
if err != nil {
return nil, err
}
digests = append(digests, digest)
}
result[table] = tableFingerprint{Count: int64(len(digests)), Digest: combineRowDigests(digests)}
}
// F01: resolve the resources once from exact binding ids and body handles
// instead of scanning every tenant resource with a correlated body filter.
rows, err := scopedResourceRows(ctx, &exporter, s.identity, true, bodies, bindings)
if err != nil {
return nil, fmt.Errorf("读取生产 resources 指纹失败: %w", err)
}
digests := make([]string, 0, len(rows))
for _, raw := range rows {
var value map[string]json.RawMessage
if err := json.Unmarshal(raw, &value); err != nil {
return nil, err
}
digest, err := canonicalRowDigest(value)
if err != nil {
return nil, err
}
digests = append(digests, digest)
}
result["resources"] = tableFingerprint{Count: int64(len(digests)), Digest: combineRowDigests(digests)}
return result, nil
}
func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
archive, err := librarysnapshot.Open(s.archiveRoot, 256*1024*1024)
if err != nil {
return libraryrelease.BackupRef{}, errors.New("归档不可用,无法创建预备份")
}
defer archive.Close()
createdAt := s.now().UTC()
before, err := s.fingerprints(ctx)
if err != nil {
return libraryrelease.BackupRef{}, err
}
// 整实例指纹:恢复之后要拿它证明"多写的、少写的都没有"。只按目标范围比,
// 写到范围外的行会被漏掉(真实事故:恢复报成功但目标多 67 行)。
universeBefore, err := universeFingerprint(ctx, s.pool)
if err != nil {
return libraryrelease.BackupRef{}, err
}
pg, err := exportLibraryBeforeImage(ctx, s.pool, s.identity)
if err != nil {
return libraryrelease.BackupRef{}, fmt.Errorf("读取生产整库失败: %w", err)
}
// Every row the actual delete predicate can reach must be present.
for _, table := range pg.Tables {
if int64(len(table.Rows)) != before[table.Name].Count {
return libraryrelease.BackupRef{}, fmt.Errorf("before-image coverage mismatch: %s", table.Name)
}
}
es, err := s.search.ExportLibrary(ctx, SearchIdentity{TenantID: s.identity.TenantID,
KnowledgeBaseID: s.identity.KnowledgeBaseID}, s.maxDocuments)
if err != nil {
return libraryrelease.BackupRef{}, fmt.Errorf("读取生产索引失败: %w", err)
}
if es.MappingSHA256 == "" {
// 生产索引不存在:只有数据库也为空时才是合法的"空基线",否则说明索引丢了。
if len(pg.Table("knowledges").Rows) != 0 || len(pg.Table("chunks").Rows) != 0 {
return libraryrelease.BackupRef{}, errors.New("生产索引缺失但数据库非空:先修索引再发布")
}
} else if es.MappingSHA256 != s.compatibility.IndexMappingSHA256 {
// 契约不符说明生产换了索引映射:此时备份出来的东西无法用同一套恢复。
// 唯一例外是"空生产":零文档零分块时映射差异不影响可恢复性。
emptyProduction := len(pg.Table("knowledges").Rows) == 0 && len(pg.Table("chunks").Rows) == 0 && len(es.Documents) == 0
if !emptyProduction {
return libraryrelease.BackupRef{}, errors.New("生产索引映射与归档契约不一致")
}
}
objectBefore, err := s.objectVersions(ctx, pg)
if err != nil {
return libraryrelease.BackupRef{}, err
}
// ArchiveLibraryCandidate 自己按 resources 行读取对象字节并计算摘要,
// 所以这里直接给它生产对象读取器,不再重复导出一次资源。
snapshotID, err := archiveLibraryRows(ctx, archive, s.scope, s.identity, s.compatibility, pg, es, s.objects, createdAt)
if err != nil {
return libraryrelease.BackupRef{}, fmt.Errorf("密封预备份失败: %w", err)
}
// 静默检查:导出期间生产内容不能变。只比行数会被"同数量改写"骗过,
// 所以这里比对逐行内容摘要(行数 + 身份 + 内容)。
after, err := s.fingerprints(ctx)
if err != nil {
return libraryrelease.BackupRef{}, err
}
for table, beforeState := range before {
if after[table] != beforeState {
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间生产 %s 内容发生变化,切点不一致", table)
}
}
universeAfter, err := universeFingerprint(ctx, s.pool)
if err != nil {
return libraryrelease.BackupRef{}, err
}
if diff := universeDiff(universeBefore, universeAfter); diff != "" {
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间整实例知识域发生变化(%s),切点不一致", diff)
}
for table, want := range current.Counts {
if table == "index" {
continue
}
if after[table].Count != want {
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间生产 %s 行数从 %d 变为 %d", table, want, after[table].Count)
}
}
manifest, err := archive.Load(snapshotID)
if err != nil {
return libraryrelease.BackupRef{}, errors.New("预备份封存后无法校验")
}
objectAfter, err := s.objectVersions(ctx, pg)
if err != nil {
return libraryrelease.BackupRef{}, err
}
for id, before := range objectBefore {
if objectAfter[id] != before {
return libraryrelease.BackupRef{}, errors.New("object changed during prebackup; inconsistent version boundary")
}
}
var total int64
for _, part := range manifest.Parts {
if part.Kind == "resource" {
observed, ok := objectBefore[part.Name]
if !ok || observed.SHA256 != part.SHA256 || observed.Bytes != part.Bytes {
return libraryrelease.BackupRef{}, errors.New("archived object differs from prebackup version boundary")
}
}
total += part.Bytes
}
return libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID, Bytes: total,
CreatedAt: createdAt, Universe: universeBefore}, nil
}
// Exact versions, not just bytes, bind the multi-store backup read interval.
// An unchanged byte string in a new version is still a different restore base.
func (s *ProductionBackupStore) objectVersions(ctx context.Context, pg PostgresSnapshot) (map[string]objectObservation, error) {
out := map[string]objectObservation{}
reader, ok := s.objects.(interface {
ObserveObject(context.Context, string, string) (objectObservation, error)
})
for _, raw := range pg.Table("resources").Rows {
if !ok {
return nil, errors.New("prebackup reader lacks exact object-version observations")
}
var row struct {
ID string `json:"id"`
Provider string `json:"provider"`
Path string `json:"physical_path"`
}
if e := json.Unmarshal(raw, &row); e != nil {
return nil, e
}
observed, e := reader.ObserveObject(ctx, row.Provider, row.Path)
if e != nil {
return nil, e
}
if !observed.Exists || observed.VersionID == "" || observed.VersionID == "null" || observed.SHA256 == "" {
return nil, errors.New("prebackup requires an existing versioned object")
}
out[row.ID] = observed
}
return out, nil
}
func (s *ProductionBackupStore) rowCounts(ctx context.Context) (map[string]int64, error) {
counts := map[string]int64{}
for _, table := range productionLibraryTables {
var count int64
if err := s.pool.QueryRow(ctx, fmt.Sprintf(`SELECT count(*) FROM %s WHERE %s`,
table, scopeWhere(table)), s.identity.TenantID, s.identity.KnowledgeBaseID).Scan(&count); err != nil {
return nil, fmt.Errorf("读取生产 %s 行数失败", table)
}
counts[table] = count
}
return counts, nil
}