Files
knowledge-governance/internal/releaseadapter/v071/library_verify_content.go
T
姚章浩 fba2713b4e
build / build (push) Successful in 1m54s
feat(library): 发布写入支持 WeKnora 0.9.0 schema 134
整库发布写入路径原先按 0.7.1 的固定表集合工作:0.9.0 没有 wiki_log_entries,
Current/清理/编目/整实例指纹都会在不存在表上失败,跨版本闸门也因快照丢掉
SchemaVersion 而失效(未知版本被当作 74)。

改为在装配时解析目标 profile,表集合与清理顺序按解析结果;快照复制与范围改写
带上 SchemaVersion/Profile;跨版本发布显式拒绝并同时点名两个版本。单文档发布包
仍只支持 0.7.1,但改为明确拒绝并说明原因。

本机隔离验证:真实 0.9.0 schema 源库→导出→封存→写入干净 134 目标→逐表核对→
写前镜像回滚;74→134 拒绝与 134 缺表不报错两个负例通过。make verify 17 项全绿。
2026-11-16 20:42:23 +08:00

353 lines
12 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"knowledge-governance/internal/librarysnapshot"
)
// 内容级校验:行数相同、ID 相同不能证明内容没变。
//
// 这里对每张知识域表取 to_jsonb 行,剔除"保存就会变"的时间戳类字段后做规范化
// 摘要;索引侧比对归档文档与生产导出的文档摘要;对象侧逐字节比对。
// 允许的易变字段只有时间戳,正文、标题、向量、路径、摘要都不在豁免之列。
var contentVolatileFields = map[string]bool{
"created_at": true,
"updated_at": true,
"processed_at": true,
}
func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) {
normalized := make(map[string]any, len(raw))
for key, value := range raw {
if contentVolatileFields[key] {
continue
}
var decoded any
decoder := json.NewDecoder(bytes.NewReader(value))
decoder.UseNumber()
if err := decoder.Decode(&decoded); err != nil {
return "", fmt.Errorf("decode column %s: %w", key, err)
}
normalized[key] = decoded
}
encoded, err := json.Marshal(normalized)
if err != nil {
return "", err
}
sum := sha256.Sum256(encoded)
return hex.EncodeToString(sum[:]), nil
}
func combineRowDigests(digests []string) string {
sort.Strings(digests)
sum := sha256.Sum256([]byte(joinLines(digests)))
return hex.EncodeToString(sum[:])
}
func joinLines(items []string) string {
joined := ""
for index, item := range items {
if index > 0 {
joined += "\n"
}
joined += item
}
return joined
}
// productionTableContent 读取生产侧规范内容摘要;只比对归档里实际存在的列。
// 归档没有的列(生产默认值/后续新增列)不参与比对,避免"我们没管的列"造成假失败,
// 但归档里有的每一列都必须逐字一致。
func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, table string, columns map[string]bool) (int64, string, error) {
identity := PostgresIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID}
exporter := PostgresExporter{db: t.pool}
var rows []json.RawMessage
if table == "resources" {
// F01: verification must use the same bounded reference set as capture;
// the legacy per-resource body scan made post-sync verification minutes
// long on a small target inside a large tenant.
var bodies, bindings []json.RawMessage
for _, name := range t.tables {
if name == "resources" {
continue
}
items, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`,
name, name, t.scopeWhere(name)), identity.TenantID, identity.KnowledgeBaseID)
if err != nil {
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
}
if name != "resource_bindings" && name != "knowledge_tag_relations" {
bodies = append(bodies, items...)
}
if name == "resource_bindings" {
bindings = items
}
}
items, err := scopedResourceRows(ctx, &exporter, identity, true, bodies, bindings)
if err != nil {
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
}
rows = items
} else {
items, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, t.scopeWhere(table)),
identity.TenantID, identity.KnowledgeBaseID)
if err != nil {
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
}
rows = items
}
digests := make([]string, 0, len(rows))
for _, raw := range rows {
var value map[string]json.RawMessage
if err := json.Unmarshal(raw, &value); err != nil {
return 0, "", err
}
projected := make(map[string]json.RawMessage, len(columns))
for key := range columns {
if value, ok := value[key]; ok {
projected[key] = value
}
}
digest, err := canonicalRowDigest(projected)
if err != nil {
return 0, "", err
}
digests = append(digests, digest)
}
return int64(len(digests)), combineRowDigests(digests), nil
}
// archivedTableContent 读取归档侧规范内容摘要,并返回归档里出现过的列集合。
func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part,
transform func(map[string]json.RawMessage) error) (int64, string, map[string]bool, error) {
reader, err := archive.OpenPart(part)
if err != nil {
return 0, "", nil, err
}
defer reader.Close()
decoder := json.NewDecoder(reader)
token, err := decoder.Token()
if err != nil || token != json.Delim('[') {
return 0, "", nil, errors.New("归档部件不是行数组")
}
digests := make([]string, 0, 64)
columns := map[string]bool{}
for decoder.More() {
var raw map[string]json.RawMessage
if err := decoder.Decode(&raw); err != nil {
return 0, "", nil, err
}
if transform != nil {
// 写入时对行做过的变换(例如资源定位符换桶)校验时必须同样应用。
if err := transform(raw); err != nil {
return 0, "", nil, err
}
}
for key := range raw {
columns[key] = true
}
digest, err := canonicalRowDigest(raw)
if err != nil {
return 0, "", nil, err
}
digests = append(digests, digest)
}
return int64(len(digests)), combineRowDigests(digests), columns, nil
}
// orphanIndexDocuments 找出"索引里有、生产分块里没有"的文档(按 chunk_id 关联)。
func (t *ProductionLibraryTarget) orphanIndexDocuments(ctx context.Context, documents []SearchDocument) ([]string, error) {
if len(documents) == 0 {
return nil, nil
}
// 按文档归属判断而不是按 chunk_id:真实 WeKnora 是父子分块,
// 索引里的子块 ID 本来就不在 chunks 表里,按 chunk_id 判会误报孤儿。
rows, err := t.pool.Query(ctx, `SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL`,
t.scope.TenantID, t.scope.KnowledgeBaseID)
if err != nil {
return nil, err
}
defer rows.Close()
known := map[string]bool{}
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
known[id] = true
}
if err := rows.Err(); err != nil {
return nil, err
}
orphans := make([]string, 0, 4)
for _, document := range documents {
var source struct {
ChunkID string `json:"chunk_id"`
KnowledgeID string `json:"knowledge_id"`
}
if json.Unmarshal(document.Source, &source) != nil || source.KnowledgeID == "" {
return nil, errors.New("index document has invalid or missing knowledge_id")
}
if !known[source.KnowledgeID] {
orphans = append(orphans, document.ID)
}
}
return orphans, nil
}
// firstIndexDifference 返回第一处不一致的文档 ID 与字段,便于定位。
func firstIndexDifference(archived, live []SearchDocument) string {
liveByID := make(map[string]json.RawMessage, len(live))
for _, document := range live {
liveByID[document.ID] = document.Source
}
seen := make(map[string]bool, len(archived))
for _, document := range archived {
seen[document.ID] = true
current, ok := liveByID[document.ID]
if !ok {
return document.ID + "(生产缺少)"
}
var want, got map[string]json.RawMessage
if json.Unmarshal(document.Source, &want) != nil || json.Unmarshal(current, &got) != nil {
return document.ID + "(无法解析)"
}
for key, value := range want {
other, exists := got[key]
if !exists {
return fmt.Sprintf("%s(生产缺少字段 %s)", document.ID, key)
}
if string(value) != string(other) {
return fmt.Sprintf("%s(字段 %s 不同:归档 %.60s 生产 %.60s)", document.ID, key, value, other)
}
}
for key := range got {
if _, exists := want[key]; !exists {
return fmt.Sprintf("%s(生产多出字段 %s)", document.ID, key)
}
}
}
for _, document := range live {
if !seen[document.ID] {
return document.ID + "(生产多余)"
}
}
return "数量一致但摘要不同"
}
// verifyIndexDigest 比对归档索引文档与生产实际索引文档的规范化摘要。
func (t *ProductionLibraryTarget) verifyIndexDigest(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
for _, part := range manifest.Parts {
if part.Kind != "index" {
continue
}
reader, err := archive.OpenPart(part)
if err != nil {
return err
}
documents, err := librarysnapshot.ReadIndexDocuments(reader)
reader.Close()
if err != nil {
return err
}
if t.mappingActive && t.targetIDs != nil {
for i := range documents {
mappedID, mappedSource, err := t.targetIDs.remapIndexDocument(documents[i].ID, documents[i].Source)
if err != nil {
return err
}
documents[i] = librarysnapshot.IndexDocument{ID: mappedID, Source: mappedSource}
}
}
archived := make([]SearchDocument, 0, len(documents))
for _, document := range documents {
// 写入时 BulK Index 会把 tenant_id/knowledge_base_id 覆盖成本目标的身份,
// 校验必须用同样的规范化,否则"写对了"也会被判不一致。
source := map[string]json.RawMessage{}
if err := json.Unmarshal(document.Source, &source); err != nil {
return errors.New("归档索引文档不是对象")
}
source["tenant_id"], _ = json.Marshal(t.scope.TenantID)
source["knowledge_base_id"], _ = json.Marshal(t.scope.KnowledgeBaseID)
encoded, err := json.Marshal(source)
if err != nil {
return err
}
canonical, err := canonicalJSON(encoded)
if err != nil {
return errors.New("归档索引文档无法规范化")
}
archived = append(archived, SearchDocument{ID: document.ID, Source: canonical})
}
// 目标 ID 映射会改变文档 ID 的排序位置;摘要按 ID 排序后的集合计算,
// 否则少一个/多一个的差异会被误报成"数量一致但摘要不同"。
sort.Slice(archived, func(i, j int) bool { return archived[i].ID < archived[j].ID })
wantDigest, err := searchDocumentsDigest(archived)
if err != nil {
return err
}
snapshot, err := t.search.ExportLibrary(ctx, SearchIdentity{TenantID: t.scope.TenantID,
KnowledgeBaseID: t.scope.KnowledgeBaseID}, t.config.MaxDocuments)
if err != nil {
return fmt.Errorf("读取生产索引失败: %w", err)
}
// 孤儿向量:索引里有、数据库分块里没有的文档必须报出来。
// 只比"归档 vs 生产"会漏掉两边一致但都错的情况。
orphans, orphanErr := t.orphanIndexDocuments(ctx, snapshot.Documents)
if orphanErr != nil {
return fmt.Errorf("index ownership verification failed: %w", orphanErr)
}
if len(orphans) > 0 {
return fmt.Errorf("生产索引存在 %d 条孤儿文档(所属文档已不在数据库),首个 %s", len(orphans), orphans[0])
}
if len(snapshot.Documents) != len(archived) || snapshot.DocumentsSHA256 != wantDigest {
// 摘要不一致时给出第一处差异文档,运维才能定位是"少了/多了/被改了"。
return fmt.Errorf("生产索引与快照不一致:文档 %d/%d,首个差异 %s",
len(snapshot.Documents), len(archived), firstIndexDifference(archived, snapshot.Documents))
}
}
return nil
}
// productionResourceContent 只读归档里列出的那些资源行:resources 是多知识库
// 共享表,按库重新推导会漏掉"只被历史上某条正文引用过"的行,恢复得一字不差
// 也会被判失败。这里按身份集合取行,再逐行做内容摘要。
func (t *ProductionLibraryTarget) productionResourceContent(ctx context.Context, ids []string, columns map[string]bool) (int64, string, error) {
rows, err := t.pool.Query(ctx, `SELECT to_jsonb(r) FROM resources r WHERE r.id = ANY($1::text[]) ORDER BY r.id`, ids)
if err != nil {
return 0, "", fmt.Errorf("读取生产 resources 内容失败: %w", err)
}
defer rows.Close()
digests := make([]string, 0, len(ids))
for rows.Next() {
var raw map[string]json.RawMessage
if err := rows.Scan(&raw); err != nil {
return 0, "", err
}
projected := make(map[string]json.RawMessage, len(columns))
for key := range columns {
if value, ok := raw[key]; ok {
projected[key] = value
}
}
digest, err := canonicalRowDigest(projected)
if err != nil {
return 0, "", err
}
digests = append(digests, digest)
}
if err := rows.Err(); err != nil {
return 0, "", err
}
return int64(len(digests)), combineRowDigests(digests), nil
}