Files
knowledge-governance/internal/releaseadapter/v071/library_verify_content.go
T
姚章浩 1a0f99e346
build / build (push) Successful in 1m44s
fix(release): 修复发布管理F01-F11资源闭包与状态呈现
F01:捕获、写前镜像、指纹和恢复后内容校验共用按目标精确绑定的资源闭包:
从已限定范围的 resource_bindings 与正文 handle 一次提取,再按索引取资源,
不再对每个租户资源扫描全部正文表。无绑定正文引用仍保留,缺失资源仍失败。

F02/F03:恢复验证使用 capture/verify/canary/restore/finalize 自己的阶段,登记
证据不再叫新版本生效;顶部优先在飞作业;已验证且 API reason 为空时不再同屏
显示未读到状态,报告可展开。

F04-F11:显示实际目标实例/项目/库;运行中主按钮查看当前任务并禁止重复启动;
快照历史默认已登记、可筛选分页并定位所选;无真实下次计划时显示未读取;各
scope 独立 overview;新操作清旧通知;存储行按文档/分块/资源/绑定及 Wiki
页面/目录/日志分项。高影响恢复验证/恢复增加一次影响确认。

验证:隔离 PG 20k 无关资源夹具旧扫描 683ms→新导出 41ms,资源 ID 闭包相等;
Go/前端全量及 make verify 17 项通过。未部署、未做 8088 UI 验收。
2026-11-05 12:50:11 +08:00

353 lines
12 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"knowledge-governance/internal/librarysnapshot"
)
// 内容级校验:行数相同、ID 相同不能证明内容没变。
//
// 这里对每张知识域表取 to_jsonb 行,剔除"保存就会变"的时间戳类字段后做规范化
// 摘要;索引侧比对归档文档与生产导出的文档摘要;对象侧逐字节比对。
// 允许的易变字段只有时间戳,正文、标题、向量、路径、摘要都不在豁免之列。
var contentVolatileFields = map[string]bool{
"created_at": true,
"updated_at": true,
"processed_at": true,
}
func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) {
normalized := make(map[string]any, len(raw))
for key, value := range raw {
if contentVolatileFields[key] {
continue
}
var decoded any
decoder := json.NewDecoder(bytes.NewReader(value))
decoder.UseNumber()
if err := decoder.Decode(&decoded); err != nil {
return "", fmt.Errorf("decode column %s: %w", key, err)
}
normalized[key] = decoded
}
encoded, err := json.Marshal(normalized)
if err != nil {
return "", err
}
sum := sha256.Sum256(encoded)
return hex.EncodeToString(sum[:]), nil
}
func combineRowDigests(digests []string) string {
sort.Strings(digests)
sum := sha256.Sum256([]byte(joinLines(digests)))
return hex.EncodeToString(sum[:])
}
func joinLines(items []string) string {
joined := ""
for index, item := range items {
if index > 0 {
joined += "\n"
}
joined += item
}
return joined
}
// productionTableContent 读取生产侧规范内容摘要;只比对归档里实际存在的列。
// 归档没有的列(生产默认值/后续新增列)不参与比对,避免"我们没管的列"造成假失败,
// 但归档里有的每一列都必须逐字一致。
func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, table string, columns map[string]bool) (int64, string, error) {
identity := PostgresIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID}
exporter := PostgresExporter{db: t.pool}
var rows []json.RawMessage
if table == "resources" {
// F01: verification must use the same bounded reference set as capture;
// the legacy per-resource body scan made post-sync verification minutes
// long on a small target inside a large tenant.
var bodies, bindings []json.RawMessage
for _, name := range productionLibraryTables {
if name == "resources" {
continue
}
items, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`,
name, name, scopeWhere(name)), identity.TenantID, identity.KnowledgeBaseID)
if err != nil {
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
}
if name != "resource_bindings" && name != "knowledge_tag_relations" {
bodies = append(bodies, items...)
}
if name == "resource_bindings" {
bindings = items
}
}
items, err := scopedResourceRows(ctx, &exporter, identity, true, bodies, bindings)
if err != nil {
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
}
rows = items
} else {
items, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table)),
identity.TenantID, identity.KnowledgeBaseID)
if err != nil {
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
}
rows = items
}
digests := make([]string, 0, len(rows))
for _, raw := range rows {
var value map[string]json.RawMessage
if err := json.Unmarshal(raw, &value); err != nil {
return 0, "", err
}
projected := make(map[string]json.RawMessage, len(columns))
for key := range columns {
if value, ok := value[key]; ok {
projected[key] = value
}
}
digest, err := canonicalRowDigest(projected)
if err != nil {
return 0, "", err
}
digests = append(digests, digest)
}
return int64(len(digests)), combineRowDigests(digests), nil
}
// archivedTableContent 读取归档侧规范内容摘要,并返回归档里出现过的列集合。
func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part,
transform func(map[string]json.RawMessage) error) (int64, string, map[string]bool, error) {
reader, err := archive.OpenPart(part)
if err != nil {
return 0, "", nil, err
}
defer reader.Close()
decoder := json.NewDecoder(reader)
token, err := decoder.Token()
if err != nil || token != json.Delim('[') {
return 0, "", nil, errors.New("归档部件不是行数组")
}
digests := make([]string, 0, 64)
columns := map[string]bool{}
for decoder.More() {
var raw map[string]json.RawMessage
if err := decoder.Decode(&raw); err != nil {
return 0, "", nil, err
}
if transform != nil {
// 写入时对行做过的变换(例如资源定位符换桶)校验时必须同样应用。
if err := transform(raw); err != nil {
return 0, "", nil, err
}
}
for key := range raw {
columns[key] = true
}
digest, err := canonicalRowDigest(raw)
if err != nil {
return 0, "", nil, err
}
digests = append(digests, digest)
}
return int64(len(digests)), combineRowDigests(digests), columns, nil
}
// orphanIndexDocuments 找出"索引里有、生产分块里没有"的文档(按 chunk_id 关联)。
func (t *ProductionLibraryTarget) orphanIndexDocuments(ctx context.Context, documents []SearchDocument) ([]string, error) {
if len(documents) == 0 {
return nil, nil
}
// 按文档归属判断而不是按 chunk_id:真实 WeKnora 是父子分块,
// 索引里的子块 ID 本来就不在 chunks 表里,按 chunk_id 判会误报孤儿。
rows, err := t.pool.Query(ctx, `SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL`,
t.scope.TenantID, t.scope.KnowledgeBaseID)
if err != nil {
return nil, err
}
defer rows.Close()
known := map[string]bool{}
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
known[id] = true
}
if err := rows.Err(); err != nil {
return nil, err
}
orphans := make([]string, 0, 4)
for _, document := range documents {
var source struct {
ChunkID string `json:"chunk_id"`
KnowledgeID string `json:"knowledge_id"`
}
if json.Unmarshal(document.Source, &source) != nil || source.KnowledgeID == "" {
return nil, errors.New("index document has invalid or missing knowledge_id")
}
if !known[source.KnowledgeID] {
orphans = append(orphans, document.ID)
}
}
return orphans, nil
}
// firstIndexDifference 返回第一处不一致的文档 ID 与字段,便于定位。
func firstIndexDifference(archived, live []SearchDocument) string {
liveByID := make(map[string]json.RawMessage, len(live))
for _, document := range live {
liveByID[document.ID] = document.Source
}
seen := make(map[string]bool, len(archived))
for _, document := range archived {
seen[document.ID] = true
current, ok := liveByID[document.ID]
if !ok {
return document.ID + "(生产缺少)"
}
var want, got map[string]json.RawMessage
if json.Unmarshal(document.Source, &want) != nil || json.Unmarshal(current, &got) != nil {
return document.ID + "(无法解析)"
}
for key, value := range want {
other, exists := got[key]
if !exists {
return fmt.Sprintf("%s(生产缺少字段 %s)", document.ID, key)
}
if string(value) != string(other) {
return fmt.Sprintf("%s(字段 %s 不同:归档 %.60s 生产 %.60s)", document.ID, key, value, other)
}
}
for key := range got {
if _, exists := want[key]; !exists {
return fmt.Sprintf("%s(生产多出字段 %s)", document.ID, key)
}
}
}
for _, document := range live {
if !seen[document.ID] {
return document.ID + "(生产多余)"
}
}
return "数量一致但摘要不同"
}
// verifyIndexDigest 比对归档索引文档与生产实际索引文档的规范化摘要。
func (t *ProductionLibraryTarget) verifyIndexDigest(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
for _, part := range manifest.Parts {
if part.Kind != "index" {
continue
}
reader, err := archive.OpenPart(part)
if err != nil {
return err
}
documents, err := librarysnapshot.ReadIndexDocuments(reader)
reader.Close()
if err != nil {
return err
}
if t.mappingActive && t.targetIDs != nil {
for i := range documents {
mappedID, mappedSource, err := t.targetIDs.remapIndexDocument(documents[i].ID, documents[i].Source)
if err != nil {
return err
}
documents[i] = librarysnapshot.IndexDocument{ID: mappedID, Source: mappedSource}
}
}
archived := make([]SearchDocument, 0, len(documents))
for _, document := range documents {
// 写入时 BulK Index 会把 tenant_id/knowledge_base_id 覆盖成本目标的身份,
// 校验必须用同样的规范化,否则"写对了"也会被判不一致。
source := map[string]json.RawMessage{}
if err := json.Unmarshal(document.Source, &source); err != nil {
return errors.New("归档索引文档不是对象")
}
source["tenant_id"], _ = json.Marshal(t.scope.TenantID)
source["knowledge_base_id"], _ = json.Marshal(t.scope.KnowledgeBaseID)
encoded, err := json.Marshal(source)
if err != nil {
return err
}
canonical, err := canonicalJSON(encoded)
if err != nil {
return errors.New("归档索引文档无法规范化")
}
archived = append(archived, SearchDocument{ID: document.ID, Source: canonical})
}
// 目标 ID 映射会改变文档 ID 的排序位置;摘要按 ID 排序后的集合计算,
// 否则少一个/多一个的差异会被误报成"数量一致但摘要不同"。
sort.Slice(archived, func(i, j int) bool { return archived[i].ID < archived[j].ID })
wantDigest, err := searchDocumentsDigest(archived)
if err != nil {
return err
}
snapshot, err := t.search.ExportLibrary(ctx, SearchIdentity{TenantID: t.scope.TenantID,
KnowledgeBaseID: t.scope.KnowledgeBaseID}, t.config.MaxDocuments)
if err != nil {
return fmt.Errorf("读取生产索引失败: %w", err)
}
// 孤儿向量:索引里有、数据库分块里没有的文档必须报出来。
// 只比"归档 vs 生产"会漏掉两边一致但都错的情况。
orphans, orphanErr := t.orphanIndexDocuments(ctx, snapshot.Documents)
if orphanErr != nil {
return fmt.Errorf("index ownership verification failed: %w", orphanErr)
}
if len(orphans) > 0 {
return fmt.Errorf("生产索引存在 %d 条孤儿文档(所属文档已不在数据库),首个 %s", len(orphans), orphans[0])
}
if len(snapshot.Documents) != len(archived) || snapshot.DocumentsSHA256 != wantDigest {
// 摘要不一致时给出第一处差异文档,运维才能定位是"少了/多了/被改了"。
return fmt.Errorf("生产索引与快照不一致:文档 %d/%d,首个差异 %s",
len(snapshot.Documents), len(archived), firstIndexDifference(archived, snapshot.Documents))
}
}
return nil
}
// productionResourceContent 只读归档里列出的那些资源行:resources 是多知识库
// 共享表,按库重新推导会漏掉"只被历史上某条正文引用过"的行,恢复得一字不差
// 也会被判失败。这里按身份集合取行,再逐行做内容摘要。
func (t *ProductionLibraryTarget) productionResourceContent(ctx context.Context, ids []string, columns map[string]bool) (int64, string, error) {
rows, err := t.pool.Query(ctx, `SELECT to_jsonb(r) FROM resources r WHERE r.id = ANY($1::text[]) ORDER BY r.id`, ids)
if err != nil {
return 0, "", fmt.Errorf("读取生产 resources 内容失败: %w", err)
}
defer rows.Close()
digests := make([]string, 0, len(ids))
for rows.Next() {
var raw map[string]json.RawMessage
if err := rows.Scan(&raw); err != nil {
return 0, "", err
}
projected := make(map[string]json.RawMessage, len(columns))
for key := range columns {
if value, ok := raw[key]; ok {
projected[key] = value
}
}
digest, err := canonicalRowDigest(projected)
if err != nil {
return 0, "", err
}
digests = append(digests, digest)
}
if err := rows.Err(); err != nil {
return 0, "", err
}
return int64(len(digests)), combineRowDigests(digests), nil
}