Files
knowledge-governance/internal/releaseadapter/v071/library_backup_store.go
T
姚章浩 ac3b395102
build / build (push) Successful in 1m21s
fix(library): 对齐真实 WeKnora Schema + 入口级真实服务验收
为什么:候选环境改用真实 WeKnora 表结构后,暴露出多处"按猜的列名/形状"的假设,
以及若干 varchar 长度、对象定位符、空索引基线问题,导致真实链路跑不通。

- knowledge_tag_relations 无 tenant_id/KB 列 → 归属走父表;resources 作用域=绑定+chunk 引用。
- 写入顺序按真实外键;无代理键表用内容摘要核对。
- 队列作业 ID / 版本记录 ID 改固定长度摘要(列宽 varchar(36)/varchar(64))。
- 对象定位符拆 storage://<backend>/oss://<bucket>/<key>;provider 可配置。
- 空索引基线全链路一致(Current/Export/Backup/Archive),非空仍严格比对映射。
- finalize 阶段:版本登记失败不报成功;崩溃接管复用已持久化预备份。
- 迁移 000069:同一目标的 sync/restore 跨 kind 互斥。
2026-10-28 17:53:43 +08:00

189 lines
7.2 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
)
// ProductionBackupStore 在生产写入前创建**独立完整预备份**:从生产读一次整库
// (PG RepeatableRead + ES 导出一致快照 + 资源对象),密封成归档里的一个版本。
//
// 为什么要有静默检查:预备份期间生产若仍在变,导出的 PG/ES/对象就不是同一个切点,
// 恢复它等于恢复到"没人见过的状态"。所以导出前后各读一次行数,不一致就失败,
// 让作业重试,而不是拿一个不一致的备份当恢复点。
type ProductionBackupStore struct {
pool *pgxpool.Pool
search *SearchExporter
objects ObjectReader
archiveRoot string
scope string
identity PostgresIdentity
compatibility librarysnapshot.Compatibility
maxDocuments int
maxObjectByte int64
now func() time.Time
}
type ProductionBackupConfig struct {
Pool *pgxpool.Pool
Search *SearchExporter
Objects ObjectReader
ArchiveRoot string
Scope string
Identity PostgresIdentity
Compatibility librarysnapshot.Compatibility
MaxDocuments int
MaxObjectByte int64
Now func() time.Time
}
func NewProductionBackupStore(config ProductionBackupConfig) (*ProductionBackupStore, error) {
if config.Pool == nil || config.Search == nil || config.Objects == nil || config.ArchiveRoot == "" ||
(config.Scope != "internal" && config.Scope != "external") || config.Identity.TenantID == 0 ||
!validSearchIdentifier(config.Identity.KnowledgeBaseID) || config.MaxDocuments < 1 {
return nil, errors.New("生产预备份配置不完整")
}
if config.MaxObjectByte <= 0 {
config.MaxObjectByte = 256 * 1024 * 1024
}
now := config.Now
if now == nil {
now = time.Now
}
return &ProductionBackupStore{pool: config.Pool, search: config.Search, objects: config.Objects,
archiveRoot: config.ArchiveRoot, scope: config.Scope, identity: config.Identity,
compatibility: config.Compatibility, maxDocuments: config.MaxDocuments,
maxObjectByte: config.MaxObjectByte, now: now}, nil
}
// tableFingerprint 是一张表的"数量 + 内容摘要"指纹:同数量改写也会被识别。
type tableFingerprint struct {
Count int64
Digest string
}
// fingerprints 读取当前生产指纹;Prepare 在导出前后各取一次做一致性判断。
func (s *ProductionBackupStore) fingerprints(ctx context.Context) (map[string]tableFingerprint, error) {
result := map[string]tableFingerprint{}
for _, table := range productionLibraryTables {
rows, err := s.pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`,
table, table, scopeWhere(table)), s.identity.TenantID, s.identity.KnowledgeBaseID)
if err != nil {
return nil, fmt.Errorf("读取生产 %s 指纹失败", table)
}
digests := make([]string, 0, 64)
for rows.Next() {
var raw map[string]json.RawMessage
if err := rows.Scan(&raw); err != nil {
rows.Close()
return nil, err
}
digest, err := canonicalRowDigest(raw)
if err != nil {
rows.Close()
return nil, err
}
digests = append(digests, digest)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
result[table] = tableFingerprint{Count: int64(len(digests)), Digest: combineRowDigests(digests)}
}
return result, nil
}
func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current libraryrelease.CurrentState) (libraryrelease.BackupRef, error) {
archive, err := librarysnapshot.Open(s.archiveRoot, 256*1024*1024)
if err != nil {
return libraryrelease.BackupRef{}, errors.New("归档不可用,无法创建预备份")
}
defer archive.Close()
createdAt := s.now().UTC()
before, err := s.fingerprints(ctx)
if err != nil {
return libraryrelease.BackupRef{}, err
}
pg, err := ExportLibrary(ctx, s.pool, s.identity)
if err != nil {
return libraryrelease.BackupRef{}, fmt.Errorf("读取生产整库失败: %w", err)
}
// 整库导出用 ExportLibrary(按租户+知识库,不带单文档身份),
// 与封存路径复用同一份实现,避免两套范围规则。
es, err := s.search.ExportLibrary(ctx, SearchIdentity{TenantID: s.identity.TenantID,
KnowledgeBaseID: s.identity.KnowledgeBaseID}, s.maxDocuments)
if err != nil {
return libraryrelease.BackupRef{}, fmt.Errorf("读取生产索引失败: %w", err)
}
if es.MappingSHA256 == "" {
// 生产索引不存在:只有数据库也为空时才是合法的"空基线",否则说明索引丢了。
if len(pg.Table("knowledges").Rows) != 0 || len(pg.Table("chunks").Rows) != 0 {
return libraryrelease.BackupRef{}, errors.New("生产索引缺失但数据库非空:先修索引再发布")
}
} else if es.MappingSHA256 != s.compatibility.IndexMappingSHA256 {
// 契约不符说明生产换了索引映射:此时备份出来的东西无法用同一套恢复。
// 唯一例外是"空生产":零文档零分块时映射差异不影响可恢复性。
emptyProduction := len(pg.Table("knowledges").Rows) == 0 && len(pg.Table("chunks").Rows) == 0 && len(es.Documents) == 0
if !emptyProduction {
return libraryrelease.BackupRef{}, errors.New("生产索引映射与归档契约不一致")
}
}
// ArchiveLibraryCandidate 自己按 resources 行读取对象字节并计算摘要,
// 所以这里直接给它生产对象读取器,不再重复导出一次资源。
snapshotID, err := ArchiveLibraryCandidate(ctx, archive, s.scope, s.identity, s.compatibility, pg, es, s.objects, createdAt)
if err != nil {
return libraryrelease.BackupRef{}, fmt.Errorf("密封预备份失败: %w", err)
}
// 静默检查:导出期间生产内容不能变。只比行数会被"同数量改写"骗过,
// 所以这里比对逐行内容摘要(行数 + 身份 + 内容)。
after, err := s.fingerprints(ctx)
if err != nil {
return libraryrelease.BackupRef{}, err
}
for table, beforeState := range before {
if after[table] != beforeState {
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间生产 %s 内容发生变化,切点不一致", table)
}
}
for table, want := range current.Counts {
if table == "index" {
continue
}
if after[table].Count != want {
return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间生产 %s 行数从 %d 变为 %d", table, want, after[table].Count)
}
}
manifest, err := archive.Load(snapshotID)
if err != nil {
return libraryrelease.BackupRef{}, errors.New("预备份封存后无法校验")
}
var total int64
for _, part := range manifest.Parts {
total += part.Bytes
}
return libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID, Bytes: total,
CreatedAt: createdAt}, nil
}
func (s *ProductionBackupStore) rowCounts(ctx context.Context) (map[string]int64, error) {
counts := map[string]int64{}
for _, table := range productionLibraryTables {
var count int64
if err := s.pool.QueryRow(ctx, fmt.Sprintf(`SELECT count(*) FROM %s WHERE %s`,
table, scopeWhere(table)), s.identity.TenantID, s.identity.KnowledgeBaseID).Scan(&count); err != nil {
return nil, fmt.Errorf("读取生产 %s 行数失败", table)
}
counts[table] = count
}
return counts, nil
}