b061a9686c
build / build (push) Successful in 1m17s
只改写入侧会让"行内容摘要"对不上:校验拿变换前的归档行与目标里的变换后行比较, 真实同步在 verify 阶段报"生产 resources 内容与快照不一致"并回滚。 把行变换抽成 rowTransform,写入与校验共用同一份。 试过:make verify 15 项全绿;真实同步复现该失败后修复(下一轮重跑验证)。
305 lines
10 KiB
Go
305 lines
10 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"sort"
|
||
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// 内容级校验:行数相同、ID 相同不能证明内容没变。
|
||
//
|
||
// 这里对每张知识域表取 to_jsonb 行,剔除"保存就会变"的时间戳类字段后做规范化
|
||
// 摘要;索引侧比对归档文档与生产导出的文档摘要;对象侧逐字节比对。
|
||
// 允许的易变字段只有时间戳,正文、标题、向量、路径、摘要都不在豁免之列。
|
||
|
||
var contentVolatileFields = map[string]bool{
|
||
"created_at": true,
|
||
"updated_at": true,
|
||
"processed_at": true,
|
||
"expires_at": true,
|
||
}
|
||
|
||
func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) {
|
||
normalized := make(map[string]any, len(raw))
|
||
for key, value := range raw {
|
||
if contentVolatileFields[key] {
|
||
continue
|
||
}
|
||
var decoded any
|
||
if err := json.Unmarshal(value, &decoded); err != nil {
|
||
return "", fmt.Errorf("decode column %s: %w", key, err)
|
||
}
|
||
normalized[key] = decoded
|
||
}
|
||
encoded, err := json.Marshal(normalized)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
sum := sha256.Sum256(encoded)
|
||
return hex.EncodeToString(sum[:]), nil
|
||
}
|
||
|
||
func combineRowDigests(digests []string) string {
|
||
sort.Strings(digests)
|
||
sum := sha256.Sum256([]byte(joinLines(digests)))
|
||
return hex.EncodeToString(sum[:])
|
||
}
|
||
|
||
func joinLines(items []string) string {
|
||
joined := ""
|
||
for index, item := range items {
|
||
if index > 0 {
|
||
joined += "\n"
|
||
}
|
||
joined += item
|
||
}
|
||
return joined
|
||
}
|
||
|
||
// productionTableContent 读取生产侧规范内容摘要;只比对归档里实际存在的列。
|
||
// 归档没有的列(生产默认值/后续新增列)不参与比对,避免"我们没管的列"造成假失败,
|
||
// 但归档里有的每一列都必须逐字一致。
|
||
func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, table string, columns map[string]bool) (int64, string, error) {
|
||
// 不加表别名:scopeWhere(resources) 里引用了 resources.id,别名会让它失效。
|
||
rows, err := t.pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table)),
|
||
t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||
if err != nil {
|
||
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
|
||
}
|
||
defer rows.Close()
|
||
digests := make([]string, 0, 64)
|
||
for rows.Next() {
|
||
var raw map[string]json.RawMessage
|
||
if err := rows.Scan(&raw); err != nil {
|
||
return 0, "", err
|
||
}
|
||
projected := make(map[string]json.RawMessage, len(columns))
|
||
for key := range columns {
|
||
if value, ok := raw[key]; ok {
|
||
projected[key] = value
|
||
}
|
||
}
|
||
digest, err := canonicalRowDigest(projected)
|
||
if err != nil {
|
||
return 0, "", err
|
||
}
|
||
digests = append(digests, digest)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return 0, "", err
|
||
}
|
||
return int64(len(digests)), combineRowDigests(digests), nil
|
||
}
|
||
|
||
// archivedTableContent 读取归档侧规范内容摘要,并返回归档里出现过的列集合。
|
||
func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part,
|
||
transform func(map[string]json.RawMessage) error) (int64, string, map[string]bool, error) {
|
||
reader, err := archive.OpenPart(part)
|
||
if err != nil {
|
||
return 0, "", nil, err
|
||
}
|
||
defer reader.Close()
|
||
decoder := json.NewDecoder(reader)
|
||
token, err := decoder.Token()
|
||
if err != nil || token != json.Delim('[') {
|
||
return 0, "", nil, errors.New("归档部件不是行数组")
|
||
}
|
||
digests := make([]string, 0, 64)
|
||
columns := map[string]bool{}
|
||
for decoder.More() {
|
||
var raw map[string]json.RawMessage
|
||
if err := decoder.Decode(&raw); err != nil {
|
||
return 0, "", nil, err
|
||
}
|
||
if transform != nil {
|
||
// 写入时对行做过的变换(例如资源定位符换桶)校验时必须同样应用。
|
||
if err := transform(raw); err != nil {
|
||
return 0, "", nil, err
|
||
}
|
||
}
|
||
for key := range raw {
|
||
columns[key] = true
|
||
}
|
||
digest, err := canonicalRowDigest(raw)
|
||
if err != nil {
|
||
return 0, "", nil, err
|
||
}
|
||
digests = append(digests, digest)
|
||
}
|
||
return int64(len(digests)), combineRowDigests(digests), columns, nil
|
||
}
|
||
|
||
// orphanIndexDocuments 找出"索引里有、生产分块里没有"的文档(按 chunk_id 关联)。
|
||
func (t *ProductionLibraryTarget) orphanIndexDocuments(ctx context.Context, documents []SearchDocument) []string {
|
||
if len(documents) == 0 {
|
||
return nil
|
||
}
|
||
// 按文档归属判断而不是按 chunk_id:真实 WeKnora 是父子分块,
|
||
// 索引里的子块 ID 本来就不在 chunks 表里,按 chunk_id 判会误报孤儿。
|
||
rows, err := t.pool.Query(ctx, `SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL`,
|
||
t.scope.TenantID, t.scope.KnowledgeBaseID)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
defer rows.Close()
|
||
known := map[string]bool{}
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil
|
||
}
|
||
known[id] = true
|
||
}
|
||
orphans := make([]string, 0, 4)
|
||
for _, document := range documents {
|
||
var source struct {
|
||
ChunkID string `json:"chunk_id"`
|
||
KnowledgeID string `json:"knowledge_id"`
|
||
}
|
||
if json.Unmarshal(document.Source, &source) != nil || source.KnowledgeID == "" {
|
||
continue
|
||
}
|
||
if !known[source.KnowledgeID] {
|
||
orphans = append(orphans, document.ID)
|
||
}
|
||
}
|
||
return orphans
|
||
}
|
||
|
||
// firstIndexDifference 返回第一处不一致的文档 ID 与字段,便于定位。
|
||
func firstIndexDifference(archived, live []SearchDocument) string {
|
||
liveByID := make(map[string]json.RawMessage, len(live))
|
||
for _, document := range live {
|
||
liveByID[document.ID] = document.Source
|
||
}
|
||
seen := make(map[string]bool, len(archived))
|
||
for _, document := range archived {
|
||
seen[document.ID] = true
|
||
current, ok := liveByID[document.ID]
|
||
if !ok {
|
||
return document.ID + "(生产缺少)"
|
||
}
|
||
var want, got map[string]json.RawMessage
|
||
if json.Unmarshal(document.Source, &want) != nil || json.Unmarshal(current, &got) != nil {
|
||
return document.ID + "(无法解析)"
|
||
}
|
||
for key, value := range want {
|
||
other, exists := got[key]
|
||
if !exists {
|
||
return fmt.Sprintf("%s(生产缺少字段 %s)", document.ID, key)
|
||
}
|
||
if string(value) != string(other) {
|
||
return fmt.Sprintf("%s(字段 %s 不同:归档 %.60s 生产 %.60s)", document.ID, key, value, other)
|
||
}
|
||
}
|
||
for key := range got {
|
||
if _, exists := want[key]; !exists {
|
||
return fmt.Sprintf("%s(生产多出字段 %s)", document.ID, key)
|
||
}
|
||
}
|
||
}
|
||
for _, document := range live {
|
||
if !seen[document.ID] {
|
||
return document.ID + "(生产多余)"
|
||
}
|
||
}
|
||
return "数量一致但摘要不同"
|
||
}
|
||
|
||
// verifyIndexDigest 比对归档索引文档与生产实际索引文档的规范化摘要。
|
||
func (t *ProductionLibraryTarget) verifyIndexDigest(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "index" {
|
||
continue
|
||
}
|
||
reader, err := archive.OpenPart(part)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
documents, err := librarysnapshot.ReadIndexDocuments(reader)
|
||
reader.Close()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
archived := make([]SearchDocument, 0, len(documents))
|
||
for _, document := range documents {
|
||
// 写入时 BulK Index 会把 tenant_id/knowledge_base_id 覆盖成本目标的身份,
|
||
// 校验必须用同样的规范化,否则"写对了"也会被判不一致。
|
||
source := map[string]json.RawMessage{}
|
||
if err := json.Unmarshal(document.Source, &source); err != nil {
|
||
return errors.New("归档索引文档不是对象")
|
||
}
|
||
source["tenant_id"], _ = json.Marshal(t.scope.TenantID)
|
||
source["knowledge_base_id"], _ = json.Marshal(t.scope.KnowledgeBaseID)
|
||
encoded, err := json.Marshal(source)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
canonical, err := canonicalJSON(encoded)
|
||
if err != nil {
|
||
return errors.New("归档索引文档无法规范化")
|
||
}
|
||
archived = append(archived, SearchDocument{ID: document.ID, Source: canonical})
|
||
}
|
||
wantDigest, err := searchDocumentsDigest(archived)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
snapshot, err := t.search.ExportLibrary(ctx, SearchIdentity{TenantID: t.scope.TenantID,
|
||
KnowledgeBaseID: t.scope.KnowledgeBaseID}, t.config.MaxDocuments)
|
||
if err != nil {
|
||
return fmt.Errorf("读取生产索引失败: %w", err)
|
||
}
|
||
// 孤儿向量:索引里有、数据库分块里没有的文档必须报出来。
|
||
// 只比"归档 vs 生产"会漏掉两边一致但都错的情况。
|
||
if orphans := t.orphanIndexDocuments(ctx, snapshot.Documents); len(orphans) > 0 {
|
||
return fmt.Errorf("生产索引存在 %d 条孤儿文档(所属文档已不在数据库),首个 %s", len(orphans), orphans[0])
|
||
}
|
||
if len(snapshot.Documents) != len(archived) || snapshot.DocumentsSHA256 != wantDigest {
|
||
// 摘要不一致时给出第一处差异文档,运维才能定位是"少了/多了/被改了"。
|
||
return fmt.Errorf("生产索引与快照不一致:文档 %d/%d,首个差异 %s",
|
||
len(snapshot.Documents), len(archived), firstIndexDifference(archived, snapshot.Documents))
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// productionResourceContent 只读归档里列出的那些资源行:resources 是多知识库
|
||
// 共享表,按库重新推导会漏掉"只被历史上某条正文引用过"的行,恢复得一字不差
|
||
// 也会被判失败。这里按身份集合取行,再逐行做内容摘要。
|
||
func (t *ProductionLibraryTarget) productionResourceContent(ctx context.Context, ids []string, columns map[string]bool) (int64, string, error) {
|
||
rows, err := t.pool.Query(ctx, `SELECT to_jsonb(r) FROM resources r WHERE r.id = ANY($1::text[]) ORDER BY r.id`, ids)
|
||
if err != nil {
|
||
return 0, "", fmt.Errorf("读取生产 resources 内容失败: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
digests := make([]string, 0, len(ids))
|
||
for rows.Next() {
|
||
var raw map[string]json.RawMessage
|
||
if err := rows.Scan(&raw); err != nil {
|
||
return 0, "", err
|
||
}
|
||
projected := make(map[string]json.RawMessage, len(columns))
|
||
for key := range columns {
|
||
if value, ok := raw[key]; ok {
|
||
projected[key] = value
|
||
}
|
||
}
|
||
digest, err := canonicalRowDigest(projected)
|
||
if err != nil {
|
||
return 0, "", err
|
||
}
|
||
digests = append(digests, digest)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return 0, "", err
|
||
}
|
||
return int64(len(digests)), combineRowDigests(digests), nil
|
||
}
|