c72b71d78a
build / build (push) Failing after 17s
为什么:父审指出预备份引用持久化失败仍继续写目标、接管依赖启动扫描且漏掉租约未过期、 恢复失败被标成"无需恢复"、版本登记与作业状态分叉。 - BackupPrepared 返回 error,持久化失败即零写入;预备份同时登记 production_prebackup 版本。 - 接管改为按队列租约判定 + 运行期 30s 扫描 + 原子抢占 + 固定幂等键 + attempt 归零。 - 接管场景保留预备份、不再宣称"未写无需恢复"。 - 恢复/重放同一快照重新激活 production_effective(清 superseded_at)。 - finalize 续作只做校验+登记(finalize_only),不重写生产。 - 索引映射基线缺失时采用首次观察值并登记;索引校验加孤儿文档检查(按 knowledge_id)。 真实验收(候选 API :18089 + 真实 PG/ES/MinIO): A0 首次发布、A1 撤销、B 按版本恢复、C 写阶段 SIGKILL 接管、 D 跨 kind 互斥与多 worker 单次执行、E1 登记失败、E2 finalize 续作、E3 预备份失败零写入。
265 lines
8.7 KiB
Go
265 lines
8.7 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"sort"
|
||
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// 内容级校验:行数相同、ID 相同不能证明内容没变。
|
||
//
|
||
// 这里对每张知识域表取 to_jsonb 行,剔除"保存就会变"的时间戳类字段后做规范化
|
||
// 摘要;索引侧比对归档文档与生产导出的文档摘要;对象侧逐字节比对。
|
||
// 允许的易变字段只有时间戳,正文、标题、向量、路径、摘要都不在豁免之列。
|
||
|
||
var contentVolatileFields = map[string]bool{
|
||
"created_at": true,
|
||
"updated_at": true,
|
||
"processed_at": true,
|
||
"expires_at": true,
|
||
}
|
||
|
||
func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) {
|
||
normalized := make(map[string]any, len(raw))
|
||
for key, value := range raw {
|
||
if contentVolatileFields[key] {
|
||
continue
|
||
}
|
||
var decoded any
|
||
if err := json.Unmarshal(value, &decoded); err != nil {
|
||
return "", fmt.Errorf("decode column %s: %w", key, err)
|
||
}
|
||
normalized[key] = decoded
|
||
}
|
||
encoded, err := json.Marshal(normalized)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(encoded)
|
||
return hex.EncodeToString(sum[:]), nil
|
||
}
|
||
|
||
func combineRowDigests(digests []string) string {
|
||
sort.Strings(digests)
|
||
sum := sha256.Sum256([]byte(joinLines(digests)))
|
||
return hex.EncodeToString(sum[:])
|
||
}
|
||
|
||
func joinLines(items []string) string {
|
||
joined := ""
|
||
for index, item := range items {
|
||
if index > 0 {
|
||
joined += "\n"
|
||
}
|
||
joined += item
|
||
}
|
||
return joined
|
||
}
|
||
|
||
// productionTableContent 读取生产侧规范内容摘要;只比对归档里实际存在的列。
|
||
// 归档没有的列(生产默认值/后续新增列)不参与比对,避免"我们没管的列"造成假失败,
|
||
// 但归档里有的每一列都必须逐字一致。
|
||
func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, table string, columns map[string]bool) (int64, string, error) {
|
||
// 不加表别名:scopeWhere(resources) 里引用了 resources.id,别名会让它失效。
|
||
rows, err := t.pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table)),
|
||
t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||
if err != nil {
|
||
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
|
||
}
|
||
defer rows.Close()
|
||
digests := make([]string, 0, 64)
|
||
for rows.Next() {
|
||
var raw map[string]json.RawMessage
|
||
if err := rows.Scan(&raw); err != nil {
|
||
return 0, "", err
|
||
}
|
||
projected := make(map[string]json.RawMessage, len(columns))
|
||
for key := range columns {
|
||
if value, ok := raw[key]; ok {
|
||
projected[key] = value
|
||
}
|
||
}
|
||
digest, err := canonicalRowDigest(projected)
|
||
if err != nil {
|
||
return 0, "", err
|
||
}
|
||
digests = append(digests, digest)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return 0, "", err
|
||
}
|
||
return int64(len(digests)), combineRowDigests(digests), nil
|
||
}
|
||
|
||
// archivedTableContent 读取归档侧规范内容摘要,并返回归档里出现过的列集合。
|
||
func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part) (int64, string, map[string]bool, error) {
|
||
reader, err := archive.OpenPart(part)
|
||
if err != nil {
|
||
return 0, "", nil, err
|
||
}
|
||
defer reader.Close()
|
||
decoder := json.NewDecoder(reader)
|
||
token, err := decoder.Token()
|
||
if err != nil || token != json.Delim('[') {
|
||
return 0, "", nil, errors.New("归档部件不是行数组")
|
||
}
|
||
digests := make([]string, 0, 64)
|
||
columns := map[string]bool{}
|
||
for decoder.More() {
|
||
var raw map[string]json.RawMessage
|
||
if err := decoder.Decode(&raw); err != nil {
|
||
return 0, "", nil, err
|
||
}
|
||
for key := range raw {
|
||
columns[key] = true
|
||
}
|
||
digest, err := canonicalRowDigest(raw)
|
||
if err != nil {
|
||
return 0, "", nil, err
|
||
}
|
||
digests = append(digests, digest)
|
||
}
|
||
return int64(len(digests)), combineRowDigests(digests), columns, nil
|
||
}
|
||
|
||
// orphanIndexDocuments 找出"索引里有、生产分块里没有"的文档(按 chunk_id 关联)。
|
||
func (t *ProductionLibraryTarget) orphanIndexDocuments(ctx context.Context, documents []SearchDocument) []string {
|
||
if len(documents) == 0 {
|
||
return nil
|
||
}
|
||
// 按文档归属判断而不是按 chunk_id:真实 WeKnora 是父子分块,
|
||
// 索引里的子块 ID 本来就不在 chunks 表里,按 chunk_id 判会误报孤儿。
|
||
rows, err := t.pool.Query(ctx, `SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL`,
|
||
t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
defer rows.Close()
|
||
known := map[string]bool{}
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil
|
||
}
|
||
known[id] = true
|
||
}
|
||
orphans := make([]string, 0, 4)
|
||
for _, document := range documents {
|
||
var source struct {
|
||
ChunkID string `json:"chunk_id"`
|
||
KnowledgeID string `json:"knowledge_id"`
|
||
}
|
||
if json.Unmarshal(document.Source, &source) != nil || source.KnowledgeID == "" {
|
||
continue
|
||
}
|
||
if !known[source.KnowledgeID] {
|
||
orphans = append(orphans, document.ID)
|
||
}
|
||
}
|
||
return orphans
|
||
}
|
||
|
||
// firstIndexDifference 返回第一处不一致的文档 ID 与字段,便于定位。
|
||
func firstIndexDifference(archived, live []SearchDocument) string {
|
||
liveByID := make(map[string]json.RawMessage, len(live))
|
||
for _, document := range live {
|
||
liveByID[document.ID] = document.Source
|
||
}
|
||
seen := make(map[string]bool, len(archived))
|
||
for _, document := range archived {
|
||
seen[document.ID] = true
|
||
current, ok := liveByID[document.ID]
|
||
if !ok {
|
||
return document.ID + "(生产缺少)"
|
||
}
|
||
var want, got map[string]json.RawMessage
|
||
if json.Unmarshal(document.Source, &want) != nil || json.Unmarshal(current, &got) != nil {
|
||
return document.ID + "(无法解析)"
|
||
}
|
||
for key, value := range want {
|
||
other, exists := got[key]
|
||
if !exists {
|
||
return fmt.Sprintf("%s(生产缺少字段 %s)", document.ID, key)
|
||
}
|
||
if string(value) != string(other) {
|
||
return fmt.Sprintf("%s(字段 %s 不同:归档 %.60s 生产 %.60s)", document.ID, key, value, other)
|
||
}
|
||
}
|
||
for key := range got {
|
||
if _, exists := want[key]; !exists {
|
||
return fmt.Sprintf("%s(生产多出字段 %s)", document.ID, key)
|
||
}
|
||
}
|
||
}
|
||
for _, document := range live {
|
||
if !seen[document.ID] {
|
||
return document.ID + "(生产多余)"
|
||
}
|
||
}
|
||
return "数量一致但摘要不同"
|
||
}
|
||
|
||
// verifyIndexDigest 比对归档索引文档与生产实际索引文档的规范化摘要。
|
||
func (t *ProductionLibraryTarget) verifyIndexDigest(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "index" {
|
||
continue
|
||
}
|
||
reader, err := archive.OpenPart(part)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
documents, err := librarysnapshot.ReadIndexDocuments(reader)
|
||
reader.Close()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
archived := make([]SearchDocument, 0, len(documents))
|
||
for _, document := range documents {
|
||
// 写入时 BulK Index 会把 tenant_id/knowledge_base_id 覆盖成本目标的身份,
|
||
// 校验必须用同样的规范化,否则"写对了"也会被判不一致。
|
||
source := map[string]json.RawMessage{}
|
||
if err := json.Unmarshal(document.Source, &source); err != nil {
|
||
return errors.New("归档索引文档不是对象")
|
||
}
|
||
source["tenant_id"], _ = json.Marshal(t.scope.TenantID)
|
||
source["knowledge_base_id"], _ = json.Marshal(t.scope.KnowledgeBaseID)
|
||
encoded, err := json.Marshal(source)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
canonical, err := canonicalJSON(encoded)
|
||
if err != nil {
|
||
return errors.New("归档索引文档无法规范化")
|
||
}
|
||
archived = append(archived, SearchDocument{ID: document.ID, Source: canonical})
|
||
}
|
||
wantDigest, err := searchDocumentsDigest(archived)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
snapshot, err := t.search.ExportLibrary(ctx, SearchIdentity{TenantID: t.scope.TenantID,
|
||
KnowledgeBaseID: t.scope.KnowledgeBaseID}, t.config.MaxDocuments)
|
||
if err != nil {
|
||
return fmt.Errorf("读取生产索引失败: %w", err)
|
||
}
|
||
// 孤儿向量:索引里有、数据库分块里没有的文档必须报出来。
|
||
// 只比"归档 vs 生产"会漏掉两边一致但都错的情况。
|
||
if orphans := t.orphanIndexDocuments(ctx, snapshot.Documents); len(orphans) > 0 {
|
||
return fmt.Errorf("生产索引存在 %d 条孤儿文档(所属文档已不在数据库),首个 %s", len(orphans), orphans[0])
|
||
}
|
||
if len(snapshot.Documents) != len(archived) || snapshot.DocumentsSHA256 != wantDigest {
|
||
// 摘要不一致时给出第一处差异文档,运维才能定位是"少了/多了/被改了"。
|
||
return fmt.Errorf("生产索引与快照不一致:文档 %d/%d,首个差异 %s",
|
||
len(snapshot.Documents), len(archived), firstIndexDifference(archived, snapshot.Documents))
|
||
}
|
||
}
|
||
return nil
|
||
}
|