0b9dbe12cf
build / build (push) Successful in 1m13s
为什么:父审指出资源范围未关联外层 resources,会把别的库/租户资源算进来; Verify 只看行数与 ID,同 ID 改正文/改向量/损坏对象都能通过;集成测试还在用假对象存储。 - scopeWhere(resources)/resourcePaths 用 resource_bindings.resource_id=resources.id 精确关联, 覆盖 knowledge/chunk 绑定并限制租户;资源部件名与资源 ID 对齐。 - Verify 增加逐行内容摘要、索引文档摘要、对象字节 SHA256 校验。 - 预备份一致性改为内容指纹(数量+摘要),同数量改写也会失败。 - 新增真实 MinIO 对象存储的本机验收用例,覆盖隔离/损坏/独立恢复。
170 lines
5.2 KiB
Go
170 lines
5.2 KiB
Go
package v071
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
)
|
|
|
|
// 内容级校验:行数相同、ID 相同不能证明内容没变。
|
|
//
|
|
// 这里对每张知识域表取 to_jsonb 行,剔除"保存就会变"的时间戳类字段后做规范化
|
|
// 摘要;索引侧比对归档文档与生产导出的文档摘要;对象侧逐字节比对。
|
|
// 允许的易变字段只有时间戳,正文、标题、向量、路径、摘要都不在豁免之列。
|
|
|
|
var contentVolatileFields = map[string]bool{
|
|
"created_at": true,
|
|
"updated_at": true,
|
|
"processed_at": true,
|
|
"expires_at": true,
|
|
}
|
|
|
|
func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) {
|
|
normalized := make(map[string]any, len(raw))
|
|
for key, value := range raw {
|
|
if contentVolatileFields[key] {
|
|
continue
|
|
}
|
|
var decoded any
|
|
if err := json.Unmarshal(value, &decoded); err != nil {
|
|
return "", fmt.Errorf("decode column %s: %w", key, err)
|
|
}
|
|
normalized[key] = decoded
|
|
}
|
|
encoded, err := json.Marshal(normalized)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
sum := sha256.Sum256(encoded)
|
|
return hex.EncodeToString(sum[:]), nil
|
|
}
|
|
|
|
func combineRowDigests(digests []string) string {
|
|
sort.Strings(digests)
|
|
sum := sha256.Sum256([]byte(joinLines(digests)))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func joinLines(items []string) string {
|
|
joined := ""
|
|
for index, item := range items {
|
|
if index > 0 {
|
|
joined += "\n"
|
|
}
|
|
joined += item
|
|
}
|
|
return joined
|
|
}
|
|
|
|
// productionTableContent 读取生产侧规范内容摘要;只比对归档里实际存在的列。
|
|
// 归档没有的列(生产默认值/后续新增列)不参与比对,避免"我们没管的列"造成假失败,
|
|
// 但归档里有的每一列都必须逐字一致。
|
|
func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, table string, columns map[string]bool) (int64, string, error) {
|
|
// 不加表别名:scopeWhere(resources) 里引用了 resources.id,别名会让它失效。
|
|
rows, err := t.pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table)),
|
|
t.scope.TenantID, t.scope.KnowledgeBaseID)
|
|
if err != nil {
|
|
return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err)
|
|
}
|
|
defer rows.Close()
|
|
digests := make([]string, 0, 64)
|
|
for rows.Next() {
|
|
var raw map[string]json.RawMessage
|
|
if err := rows.Scan(&raw); err != nil {
|
|
return 0, "", err
|
|
}
|
|
projected := make(map[string]json.RawMessage, len(columns))
|
|
for key := range columns {
|
|
if value, ok := raw[key]; ok {
|
|
projected[key] = value
|
|
}
|
|
}
|
|
digest, err := canonicalRowDigest(projected)
|
|
if err != nil {
|
|
return 0, "", err
|
|
}
|
|
digests = append(digests, digest)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return 0, "", err
|
|
}
|
|
return int64(len(digests)), combineRowDigests(digests), nil
|
|
}
|
|
|
|
// archivedTableContent 读取归档侧规范内容摘要,并返回归档里出现过的列集合。
|
|
func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part) (int64, string, map[string]bool, error) {
|
|
reader, err := archive.OpenPart(part)
|
|
if err != nil {
|
|
return 0, "", nil, err
|
|
}
|
|
defer reader.Close()
|
|
decoder := json.NewDecoder(reader)
|
|
token, err := decoder.Token()
|
|
if err != nil || token != json.Delim('[') {
|
|
return 0, "", nil, errors.New("归档部件不是行数组")
|
|
}
|
|
digests := make([]string, 0, 64)
|
|
columns := map[string]bool{}
|
|
for decoder.More() {
|
|
var raw map[string]json.RawMessage
|
|
if err := decoder.Decode(&raw); err != nil {
|
|
return 0, "", nil, err
|
|
}
|
|
for key := range raw {
|
|
columns[key] = true
|
|
}
|
|
digest, err := canonicalRowDigest(raw)
|
|
if err != nil {
|
|
return 0, "", nil, err
|
|
}
|
|
digests = append(digests, digest)
|
|
}
|
|
return int64(len(digests)), combineRowDigests(digests), columns, nil
|
|
}
|
|
|
|
// verifyIndexDigest 比对归档索引文档与生产实际索引文档的规范化摘要。
|
|
func (t *ProductionLibraryTarget) verifyIndexDigest(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error {
|
|
for _, part := range manifest.Parts {
|
|
if part.Kind != "index" {
|
|
continue
|
|
}
|
|
reader, err := archive.OpenPart(part)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
documents, err := librarysnapshot.ReadIndexDocuments(reader)
|
|
reader.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
archived := make([]SearchDocument, 0, len(documents))
|
|
for _, document := range documents {
|
|
canonical, err := canonicalJSON(document.Source)
|
|
if err != nil {
|
|
return errors.New("归档索引文档无法规范化")
|
|
}
|
|
archived = append(archived, SearchDocument{ID: document.ID, Source: canonical})
|
|
}
|
|
wantDigest, err := searchDocumentsDigest(archived)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
snapshot, err := t.search.ExportLibrary(ctx, SearchIdentity{TenantID: t.scope.TenantID,
|
|
KnowledgeBaseID: t.scope.KnowledgeBaseID}, t.config.MaxDocuments)
|
|
if err != nil {
|
|
return fmt.Errorf("读取生产索引失败: %w", err)
|
|
}
|
|
if len(snapshot.Documents) != len(archived) || snapshot.DocumentsSHA256 != wantDigest {
|
|
return fmt.Errorf("生产索引与快照不一致:文档 %d/%d,内容摘要不一致",
|
|
len(snapshot.Documents), len(archived))
|
|
}
|
|
}
|
|
return nil
|
|
}
|