Files
knowledge-governance/internal/releaseadapter/v071/library_verify_content.go
T
姚章浩 b061a9686c
build / build (push) Successful in 1m17s
fix(v071): 资源换桶的变换在校验时同用
只改写入侧会让"行内容摘要"对不上:校验拿变换前的归档行与目标里的变换后行比较,
真实同步在 verify 阶段报"生产 resources 内容与快照不一致"并回滚。
把行变换抽成 rowTransform,写入与校验共用同一份。

试过:make verify 15 项全绿;真实同步复现该失败后修复(下一轮重跑验证)。
2026-10-29 06:24:14 +08:00

305 lines
10 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"knowledge-governance/internal/librarysnapshot"
)
// 内容级校验:行数相同、ID 相同不能证明内容没变。
//
// 这里对每张知识域表取 to_jsonb 行,剔除"保存就会变"的时间戳类字段后做规范化
// 摘要;索引侧比对归档文档与生产导出的文档摘要;对象侧逐字节比对。
// 允许的易变字段只有时间戳,正文、标题、向量、路径、摘要都不在豁免之列。
var contentVolatileFields = map[string]bool{
"created_at": true,
"updated_at": true,
"processed_at": true,
"expires_at": true,
}
func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) {
normalized := make(map[string]any, len(raw))
for key, value := range raw {
if contentVolatileFields[key] {
continue
}
var decoded any
if err := json.Unmarshal(value, &decoded); err != nil {
return "", fmt.Errorf("decode column %s: %w", key, err)
}
normalized[key] = decoded
}
encoded, err := json.Marshal(normalized)
if err != nil {
return "", err
}
sum := sha256.Sum256(encoded)
return hex.EncodeToString(sum[:]), nil
}
func combineRowDigests(digests []string) string {
sort.Strings(digests)
sum := sha256.Sum256([]byte(joinLines(digests)))
return hex.EncodeToString(sum[:])
}
func joinLines(items []string) string {
joined := ""
for index, item := range items {
if index > 0 {
joined += "\n"
}
joined += item
}
return joined
}
// productionTableContent 读取生产侧规范内容摘要;只比对归档里实际存在的列。
// 归档没有的列(生产默认值/后续新增列)不参与比对,避免"我们没管的列"造成假失败,
// 但归档里有的每一列都必须逐字一致。
func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, table string, columns map[string]bool) (int64, string, error) {
// 不加表别名:scopeWhere(resources) 里引用了 resources.id,别名会让它失效。
rows, err := t.pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table)),
t.scope.TenantID, t.scope.KnowledgeBaseID)
if err != nil {
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
}
defer rows.Close()
digests := make([]string, 0, 64)
for rows.Next() {
var raw map[string]json.RawMessage
if err := rows.Scan(&raw); err != nil {
return 0, "", err
}
projected := make(map[string]json.RawMessage, len(columns))
for key := range columns {
if value, ok := raw[key]; ok {
projected[key] = value
}
}
digest, err := canonicalRowDigest(projected)
if err != nil {
return 0, "", err
}
digests = append(digests, digest)
}
if err := rows.Err(); err != nil {
return 0, "", err
}
return int64(len(digests)), combineRowDigests(digests), nil
}
// archivedTableContent 读取归档侧规范内容摘要,并返回归档里出现过的列集合。
func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part,
transform func(map[string]json.RawMessage) error) (int64, string, map[string]bool, error) {
reader, err := archive.OpenPart(part)
if err != nil {
return 0, "", nil, err
}
defer reader.Close()
decoder := json.NewDecoder(reader)
token, err := decoder.Token()
if err != nil || token != json.Delim('[') {
return 0, "", nil, errors.New("归档部件不是行数组")
}
digests := make([]string, 0, 64)
columns := map[string]bool{}
for decoder.More() {
var raw map[string]json.RawMessage
if err := decoder.Decode(&raw); err != nil {
return 0, "", nil, err
}
if transform != nil {
// 写入时对行做过的变换(例如资源定位符换桶)校验时必须同样应用。
if err := transform(raw); err != nil {
return 0, "", nil, err
}
}
for key := range raw {
columns[key] = true
}
digest, err := canonicalRowDigest(raw)
if err != nil {
return 0, "", nil, err
}
digests = append(digests, digest)
}
return int64(len(digests)), combineRowDigests(digests), columns, nil
}
// orphanIndexDocuments 找出"索引里有、生产分块里没有"的文档(按 chunk_id 关联)。
func (t *ProductionLibraryTarget) orphanIndexDocuments(ctx context.Context, documents []SearchDocument) []string {
if len(documents) == 0 {
return nil
}
// 按文档归属判断而不是按 chunk_id:真实 WeKnora 是父子分块,
// 索引里的子块 ID 本来就不在 chunks 表里,按 chunk_id 判会误报孤儿。
rows, err := t.pool.Query(ctx, `SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL`,
t.scope.TenantID, t.scope.KnowledgeBaseID)
if err != nil {
return nil
}
defer rows.Close()
known := map[string]bool{}
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil
}
known[id] = true
}
orphans := make([]string, 0, 4)
for _, document := range documents {
var source struct {
ChunkID string `json:"chunk_id"`
KnowledgeID string `json:"knowledge_id"`
}
if json.Unmarshal(document.Source, &source) != nil || source.KnowledgeID == "" {
continue
}
if !known[source.KnowledgeID] {
orphans = append(orphans, document.ID)
}
}
return orphans
}
// firstIndexDifference 返回第一处不一致的文档 ID 与字段,便于定位。
func firstIndexDifference(archived, live []SearchDocument) string {
liveByID := make(map[string]json.RawMessage, len(live))
for _, document := range live {
liveByID[document.ID] = document.Source
}
seen := make(map[string]bool, len(archived))
for _, document := range archived {
seen[document.ID] = true
current, ok := liveByID[document.ID]
if !ok {
return document.ID + "(生产缺少)"
}
var want, got map[string]json.RawMessage
if json.Unmarshal(document.Source, &want) != nil || json.Unmarshal(current, &got) != nil {
return document.ID + "(无法解析)"
}
for key, value := range want {
other, exists := got[key]
if !exists {
return fmt.Sprintf("%s(生产缺少字段 %s)", document.ID, key)
}
if string(value) != string(other) {
return fmt.Sprintf("%s(字段 %s 不同:归档 %.60s 生产 %.60s)", document.ID, key, value, other)
}
}
for key := range got {
if _, exists := want[key]; !exists {
return fmt.Sprintf("%s(生产多出字段 %s)", document.ID, key)
}
}
}
for _, document := range live {
if !seen[document.ID] {
return document.ID + "(生产多余)"
}
}
return "数量一致但摘要不同"
}
// verifyIndexDigest 比对归档索引文档与生产实际索引文档的规范化摘要。
func (t *ProductionLibraryTarget) verifyIndexDigest(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
for _, part := range manifest.Parts {
if part.Kind != "index" {
continue
}
reader, err := archive.OpenPart(part)
if err != nil {
return err
}
documents, err := librarysnapshot.ReadIndexDocuments(reader)
reader.Close()
if err != nil {
return err
}
archived := make([]SearchDocument, 0, len(documents))
for _, document := range documents {
// 写入时 BulK Index 会把 tenant_id/knowledge_base_id 覆盖成本目标的身份,
// 校验必须用同样的规范化,否则"写对了"也会被判不一致。
source := map[string]json.RawMessage{}
if err := json.Unmarshal(document.Source, &source); err != nil {
return errors.New("归档索引文档不是对象")
}
source["tenant_id"], _ = json.Marshal(t.scope.TenantID)
source["knowledge_base_id"], _ = json.Marshal(t.scope.KnowledgeBaseID)
encoded, err := json.Marshal(source)
if err != nil {
return err
}
canonical, err := canonicalJSON(encoded)
if err != nil {
return errors.New("归档索引文档无法规范化")
}
archived = append(archived, SearchDocument{ID: document.ID, Source: canonical})
}
wantDigest, err := searchDocumentsDigest(archived)
if err != nil {
return err
}
snapshot, err := t.search.ExportLibrary(ctx, SearchIdentity{TenantID: t.scope.TenantID,
KnowledgeBaseID: t.scope.KnowledgeBaseID}, t.config.MaxDocuments)
if err != nil {
return fmt.Errorf("读取生产索引失败: %w", err)
}
// 孤儿向量:索引里有、数据库分块里没有的文档必须报出来。
// 只比"归档 vs 生产"会漏掉两边一致但都错的情况。
if orphans := t.orphanIndexDocuments(ctx, snapshot.Documents); len(orphans) > 0 {
return fmt.Errorf("生产索引存在 %d 条孤儿文档(所属文档已不在数据库),首个 %s", len(orphans), orphans[0])
}
if len(snapshot.Documents) != len(archived) || snapshot.DocumentsSHA256 != wantDigest {
// 摘要不一致时给出第一处差异文档,运维才能定位是"少了/多了/被改了"。
return fmt.Errorf("生产索引与快照不一致:文档 %d/%d,首个差异 %s",
len(snapshot.Documents), len(archived), firstIndexDifference(archived, snapshot.Documents))
}
}
return nil
}
// productionResourceContent 只读归档里列出的那些资源行:resources 是多知识库
// 共享表,按库重新推导会漏掉"只被历史上某条正文引用过"的行,恢复得一字不差
// 也会被判失败。这里按身份集合取行,再逐行做内容摘要。
func (t *ProductionLibraryTarget) productionResourceContent(ctx context.Context, ids []string, columns map[string]bool) (int64, string, error) {
rows, err := t.pool.Query(ctx, `SELECT to_jsonb(r) FROM resources r WHERE r.id = ANY($1::text[]) ORDER BY r.id`, ids)
if err != nil {
return 0, "", fmt.Errorf("读取生产 resources 内容失败: %w", err)
}
defer rows.Close()
digests := make([]string, 0, len(ids))
for rows.Next() {
var raw map[string]json.RawMessage
if err := rows.Scan(&raw); err != nil {
return 0, "", err
}
projected := make(map[string]json.RawMessage, len(columns))
for key := range columns {
if value, ok := raw[key]; ok {
projected[key] = value
}
}
digest, err := canonicalRowDigest(projected)
if err != nil {
return 0, "", err
}
digests = append(digests, digest)
}
if err := rows.Err(); err != nil {
return 0, "", err
}
return int64(len(digests)), combineRowDigests(digests), nil
}