Files
knowledge-governance/internal/releaseadapter/v071/library_verify_content.go
T
姚章浩 0b9dbe12cf
build / build (push) Successful in 1m13s
fix(library): 资源范围精确关联 + 内容级校验 + 真实对象存储验收
为什么:父审指出资源范围未关联外层 resources,会把别的库/租户资源算进来;
Verify 只看行数与 ID,同 ID 改正文/改向量/损坏对象都能通过;集成测试还在用假对象存储。

- scopeWhere(resources)/resourcePaths 用 resource_bindings.resource_id=resources.id 精确关联,
  覆盖 knowledge/chunk 绑定并限制租户;资源部件名与资源 ID 对齐。
- Verify 增加逐行内容摘要、索引文档摘要、对象字节 SHA256 校验。
- 预备份一致性改为内容指纹(数量+摘要),同数量改写也会失败。
- 新增真实 MinIO 对象存储的本机验收用例,覆盖隔离/损坏/独立恢复。
2026-10-28 16:23:28 +08:00

170 lines
5.2 KiB
Go

package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"knowledge-governance/internal/librarysnapshot"
)
// 内容级校验:行数相同、ID 相同不能证明内容没变。
//
// 这里对每张知识域表取 to_jsonb 行,剔除"保存就会变"的时间戳类字段后做规范化
// 摘要;索引侧比对归档文档与生产导出的文档摘要;对象侧逐字节比对。
// 允许的易变字段只有时间戳,正文、标题、向量、路径、摘要都不在豁免之列。
var contentVolatileFields = map[string]bool{
"created_at": true,
"updated_at": true,
"processed_at": true,
"expires_at": true,
}
func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) {
normalized := make(map[string]any, len(raw))
for key, value := range raw {
if contentVolatileFields[key] {
continue
}
var decoded any
if err := json.Unmarshal(value, &decoded); err != nil {
return "", fmt.Errorf("decode column %s: %w", key, err)
}
normalized[key] = decoded
}
encoded, err := json.Marshal(normalized)
if err != nil {
return "", err
}
sum := sha256.Sum256(encoded)
return hex.EncodeToString(sum[:]), nil
}
func combineRowDigests(digests []string) string {
sort.Strings(digests)
sum := sha256.Sum256([]byte(joinLines(digests)))
return hex.EncodeToString(sum[:])
}
func joinLines(items []string) string {
joined := ""
for index, item := range items {
if index > 0 {
joined += "\n"
}
joined += item
}
return joined
}
// productionTableContent 读取生产侧规范内容摘要;只比对归档里实际存在的列。
// 归档没有的列(生产默认值/后续新增列)不参与比对,避免"我们没管的列"造成假失败,
// 但归档里有的每一列都必须逐字一致。
func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, table string, columns map[string]bool) (int64, string, error) {
// 不加表别名:scopeWhere(resources) 里引用了 resources.id,别名会让它失效。
rows, err := t.pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table)),
t.scope.TenantID, t.scope.KnowledgeBaseID)
if err != nil {
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
}
defer rows.Close()
digests := make([]string, 0, 64)
for rows.Next() {
var raw map[string]json.RawMessage
if err := rows.Scan(&raw); err != nil {
return 0, "", err
}
projected := make(map[string]json.RawMessage, len(columns))
for key := range columns {
if value, ok := raw[key]; ok {
projected[key] = value
}
}
digest, err := canonicalRowDigest(projected)
if err != nil {
return 0, "", err
}
digests = append(digests, digest)
}
if err := rows.Err(); err != nil {
return 0, "", err
}
return int64(len(digests)), combineRowDigests(digests), nil
}
// archivedTableContent 读取归档侧规范内容摘要,并返回归档里出现过的列集合。
func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part) (int64, string, map[string]bool, error) {
reader, err := archive.OpenPart(part)
if err != nil {
return 0, "", nil, err
}
defer reader.Close()
decoder := json.NewDecoder(reader)
token, err := decoder.Token()
if err != nil || token != json.Delim('[') {
return 0, "", nil, errors.New("归档部件不是行数组")
}
digests := make([]string, 0, 64)
columns := map[string]bool{}
for decoder.More() {
var raw map[string]json.RawMessage
if err := decoder.Decode(&raw); err != nil {
return 0, "", nil, err
}
for key := range raw {
columns[key] = true
}
digest, err := canonicalRowDigest(raw)
if err != nil {
return 0, "", nil, err
}
digests = append(digests, digest)
}
return int64(len(digests)), combineRowDigests(digests), columns, nil
}
// verifyIndexDigest 比对归档索引文档与生产实际索引文档的规范化摘要。
func (t *ProductionLibraryTarget) verifyIndexDigest(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
for _, part := range manifest.Parts {
if part.Kind != "index" {
continue
}
reader, err := archive.OpenPart(part)
if err != nil {
return err
}
documents, err := librarysnapshot.ReadIndexDocuments(reader)
reader.Close()
if err != nil {
return err
}
archived := make([]SearchDocument, 0, len(documents))
for _, document := range documents {
canonical, err := canonicalJSON(document.Source)
if err != nil {
return errors.New("归档索引文档无法规范化")
}
archived = append(archived, SearchDocument{ID: document.ID, Source: canonical})
}
wantDigest, err := searchDocumentsDigest(archived)
if err != nil {
return err
}
snapshot, err := t.search.ExportLibrary(ctx, SearchIdentity{TenantID: t.scope.TenantID,
KnowledgeBaseID: t.scope.KnowledgeBaseID}, t.config.MaxDocuments)
if err != nil {
return fmt.Errorf("读取生产索引失败: %w", err)
}
if len(snapshot.Documents) != len(archived) || snapshot.DocumentsSHA256 != wantDigest {
return fmt.Errorf("生产索引与快照不一致:文档 %d/%d,内容摘要不一致",
len(snapshot.Documents), len(archived))
}
}
return nil
}