ac3b395102
build / build (push) Successful in 1m21s
为什么:候选环境改用真实 WeKnora 表结构后,暴露出多处"按猜的列名/形状"的假设, 以及若干 varchar 长度、对象定位符、空索引基线问题,导致真实链路跑不通。 - knowledge_tag_relations 无 tenant_id/KB 列 → 归属走父表;resources 作用域=绑定+chunk 引用。 - 写入顺序按真实外键;无代理键表用内容摘要核对。 - 队列作业 ID / 版本记录 ID 改固定长度摘要(列宽 varchar(36)/varchar(64))。 - 对象定位符拆 storage://<backend>/oss://<bucket>/<key>;provider 可配置。 - 空索引基线全链路一致(Current/Export/Backup/Archive),非空仍严格比对映射。 - finalize 阶段:版本登记失败不报成功;崩溃接管复用已持久化预备份。 - 迁移 000069:同一目标的 sync/restore 跨 kind 互斥。
189 lines
7.2 KiB
Go
189 lines
7.2 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"knowledge-governance/internal/libraryrelease"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// ProductionBackupStore 在生产写入前创建**独立完整预备份**:从生产读一次整库
|
||
// (PG RepeatableRead + ES 导出一致快照 + 资源对象),密封成归档里的一个版本。
|
||
//
|
||
// 为什么要有静默检查:预备份期间生产若仍在变,导出的 PG/ES/对象就不是同一个切点,
|
||
// 恢复它等于恢复到"没人见过的状态"。所以导出前后各读一次行数,不一致就失败,
|
||
// 让作业重试,而不是拿一个不一致的备份当恢复点。
|
||
type ProductionBackupStore struct {
|
||
pool *pgxpool.Pool
|
||
search *SearchExporter
|
||
objects ObjectReader
|
||
archiveRoot string
|
||
scope string
|
||
identity PostgresIdentity
|
||
compatibility librarysnapshot.Compatibility
|
||
maxDocuments int
|
||
maxObjectByte int64
|
||
now func() time.Time
|
||
}
|
||
|
||
type ProductionBackupConfig struct {
|
||
Pool *pgxpool.Pool
|
||
Search *SearchExporter
|
||
Objects ObjectReader
|
||
ArchiveRoot string
|
||
Scope string
|
||
Identity PostgresIdentity
|
||
Compatibility librarysnapshot.Compatibility
|
||
MaxDocuments int
|
||
MaxObjectByte int64
|
||
Now func() time.Time
|
||
}
|
||
|
||
func NewProductionBackupStore(config ProductionBackupConfig) (*ProductionBackupStore, error) {
|
||
if config.Pool == nil || config.Search == nil || config.Objects == nil || config.ArchiveRoot == "" ||
|
||
(config.Scope != "internal" && config.Scope != "external") || config.Identity.TenantID == 0 ||
|
||
!validSearchIdentifier(config.Identity.KnowledgeBaseID) || config.MaxDocuments < 1 {
|
||
return nil, errors.New("生产预备份配置不完整")
|
||
}
|
||
if config.MaxObjectByte <= 0 {
|
||
config.MaxObjectByte = 256 * 1024 * 1024
|
||
}
|
||
now := config.Now
|
||
if now == nil {
|
||
now = time.Now
|
||
}
|
||
return &ProductionBackupStore{pool: config.Pool, search: config.Search, objects: config.Objects,
|
||
archiveRoot: config.ArchiveRoot, scope: config.Scope, identity: config.Identity,
|
||
compatibility: config.Compatibility, maxDocuments: config.MaxDocuments,
|
||
maxObjectByte: config.MaxObjectByte, now: now}, nil
|
||
}
|
||
|
||
// tableFingerprint 是一张表的"数量 + 内容摘要"指纹:同数量改写也会被识别。
|
||
type tableFingerprint struct {
|
||
Count int64
|
||
Digest string
|
||
}
|
||
|
||
// fingerprints 读取当前生产指纹;Prepare 在导出前后各取一次做一致性判断。
|
||
func (s *ProductionBackupStore) fingerprints(ctx context.Context) (map[string]tableFingerprint, error) {
|
||
result := map[string]tableFingerprint{}
|
||
for _, table := range productionLibraryTables {
|
||
rows, err := s.pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`,
|
||
table, table, scopeWhere(table)), s.identity.TenantID, s.identity.KnowledgeBaseID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取生产 %s 指纹失败", table)
|
||
}
|
||
digests := make([]string, 0, 64)
|
||
for rows.Next() {
|
||
var raw map[string]json.RawMessage
|
||
if err := rows.Scan(&raw); err != nil {
|
||
rows.Close()
|
||
return nil, err
|
||
}
|
||
digest, err := canonicalRowDigest(raw)
|
||
if err != nil {
|
||
rows.Close()
|
||
return nil, err
|
||
}
|
||
digests = append(digests, digest)
|
||
}
|
||
rows.Close()
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
result[table] = tableFingerprint{Count: int64(len(digests)), Digest: combineRowDigests(digests)}
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
|
||
archive, err := librarysnapshot.Open(s.archiveRoot, 256*1024*1024)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, errors.New("归档不可用,无法创建预备份")
|
||
}
|
||
defer archive.Close()
|
||
createdAt := s.now().UTC()
|
||
before, err := s.fingerprints(ctx)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, err
|
||
}
|
||
pg, err := ExportLibrary(ctx, s.pool, s.identity)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("读取生产整库失败: %w", err)
|
||
}
|
||
// 整库导出用 ExportLibrary(按租户+知识库,不带单文档身份),
|
||
// 与封存路径复用同一份实现,避免两套范围规则。
|
||
es, err := s.search.ExportLibrary(ctx, SearchIdentity{TenantID: s.identity.TenantID,
|
||
KnowledgeBaseID: s.identity.KnowledgeBaseID}, s.maxDocuments)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("读取生产索引失败: %w", err)
|
||
}
|
||
if es.MappingSHA256 == "" {
|
||
// 生产索引不存在:只有数据库也为空时才是合法的"空基线",否则说明索引丢了。
|
||
if len(pg.Table("knowledges").Rows) != 0 || len(pg.Table("chunks").Rows) != 0 {
|
||
return libraryrelease.BackupRef{}, errors.New("生产索引缺失但数据库非空:先修索引再发布")
|
||
}
|
||
} else if es.MappingSHA256 != s.compatibility.IndexMappingSHA256 {
|
||
// 契约不符说明生产换了索引映射:此时备份出来的东西无法用同一套恢复。
|
||
// 唯一例外是"空生产":零文档零分块时映射差异不影响可恢复性。
|
||
emptyProduction := len(pg.Table("knowledges").Rows) == 0 && len(pg.Table("chunks").Rows) == 0 && len(es.Documents) == 0
|
||
if !emptyProduction {
|
||
return libraryrelease.BackupRef{}, errors.New("生产索引映射与归档契约不一致")
|
||
}
|
||
}
|
||
// ArchiveLibraryCandidate 自己按 resources 行读取对象字节并计算摘要,
|
||
// 所以这里直接给它生产对象读取器,不再重复导出一次资源。
|
||
snapshotID, err := ArchiveLibraryCandidate(ctx, archive, s.scope, s.identity, s.compatibility, pg, es, s.objects, createdAt)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("密封预备份失败: %w", err)
|
||
}
|
||
// 静默检查:导出期间生产内容不能变。只比行数会被"同数量改写"骗过,
|
||
// 所以这里比对逐行内容摘要(行数 + 身份 + 内容)。
|
||
after, err := s.fingerprints(ctx)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, err
|
||
}
|
||
for table, beforeState := range before {
|
||
if after[table] != beforeState {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间生产 %s 内容发生变化,切点不一致", table)
|
||
}
|
||
}
|
||
for table, want := range current.Counts {
|
||
if table == "index" {
|
||
continue
|
||
}
|
||
if after[table].Count != want {
|
||
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间生产 %s 行数从 %d 变为 %d", table, want, after[table].Count)
|
||
}
|
||
}
|
||
manifest, err := archive.Load(snapshotID)
|
||
if err != nil {
|
||
return libraryrelease.BackupRef{}, errors.New("预备份封存后无法校验")
|
||
}
|
||
var total int64
|
||
for _, part := range manifest.Parts {
|
||
total += part.Bytes
|
||
}
|
||
return libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID, Bytes: total,
|
||
CreatedAt: createdAt}, nil
|
||
}
|
||
|
||
func (s *ProductionBackupStore) rowCounts(ctx context.Context) (map[string]int64, error) {
|
||
counts := map[string]int64{}
|
||
for _, table := range productionLibraryTables {
|
||
var count int64
|
||
if err := s.pool.QueryRow(ctx, fmt.Sprintf(`SELECT count(*) FROM %s WHERE %s`,
|
||
table, scopeWhere(table)), s.identity.TenantID, s.identity.KnowledgeBaseID).Scan(&count); err != nil {
|
||
return nil, fmt.Errorf("读取生产 %s 行数失败", table)
|
||
}
|
||
counts[table] = count
|
||
}
|
||
return counts, nil
|
||
}
|