package v071 import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "sort" "knowledge-governance/internal/librarysnapshot" ) // 内容级校验:行数相同、ID 相同不能证明内容没变。 // // 这里对每张知识域表取 to_jsonb 行,剔除"保存就会变"的时间戳类字段后做规范化 // 摘要;索引侧比对归档文档与生产导出的文档摘要;对象侧逐字节比对。 // 允许的易变字段只有时间戳,正文、标题、向量、路径、摘要都不在豁免之列。 var contentVolatileFields = map[string]bool{ "created_at": true, "updated_at": true, "processed_at": true, "expires_at": true, } func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) { normalized := make(map[string]any, len(raw)) for key, value := range raw { if contentVolatileFields[key] { continue } var decoded any if err := json.Unmarshal(value, &decoded); err != nil { return "", fmt.Errorf("decode column %s: %w", key, err) } normalized[key] = decoded } encoded, err := json.Marshal(normalized) if err != nil { return "", err } sum := sha256.Sum256(encoded) return hex.EncodeToString(sum[:]), nil } func combineRowDigests(digests []string) string { sort.Strings(digests) sum := sha256.Sum256([]byte(joinLines(digests))) return hex.EncodeToString(sum[:]) } func joinLines(items []string) string { joined := "" for index, item := range items { if index > 0 { joined += "\n" } joined += item } return joined } // productionTableContent 读取生产侧规范内容摘要;只比对归档里实际存在的列。 // 归档没有的列(生产默认值/后续新增列)不参与比对,避免"我们没管的列"造成假失败, // 但归档里有的每一列都必须逐字一致。 func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, table string, columns map[string]bool) (int64, string, error) { // 不加表别名:scopeWhere(resources) 里引用了 resources.id,别名会让它失效。 rows, err := t.pool.Query(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table)), t.scope.TenantID, t.scope.KnowledgeBaseID) if err != nil { return 0, "", fmt.Errorf("读取生产 %s 内容失败: %w", table, err) } defer rows.Close() digests := make([]string, 0, 64) for rows.Next() { var raw map[string]json.RawMessage if err := rows.Scan(&raw); err != nil { return 0, "", err } projected := make(map[string]json.RawMessage, len(columns)) for key := range columns { if value, ok := raw[key]; ok { projected[key] = value } } digest, err := canonicalRowDigest(projected) if err != nil { return 0, "", err } digests = append(digests, digest) } if err := rows.Err(); err != nil { return 0, "", err } return int64(len(digests)), combineRowDigests(digests), nil } // archivedTableContent 读取归档侧规范内容摘要,并返回归档里出现过的列集合。 func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part, transform func(map[string]json.RawMessage) error) (int64, string, map[string]bool, error) { reader, err := archive.OpenPart(part) if err != nil { return 0, "", nil, err } defer reader.Close() decoder := json.NewDecoder(reader) token, err := decoder.Token() if err != nil || token != json.Delim('[') { return 0, "", nil, errors.New("归档部件不是行数组") } digests := make([]string, 0, 64) columns := map[string]bool{} for decoder.More() { var raw map[string]json.RawMessage if err := decoder.Decode(&raw); err != nil { return 0, "", nil, err } if transform != nil { // 写入时对行做过的变换(例如资源定位符换桶)校验时必须同样应用。 if err := transform(raw); err != nil { return 0, "", nil, err } } for key := range raw { columns[key] = true } digest, err := canonicalRowDigest(raw) if err != nil { return 0, "", nil, err } digests = append(digests, digest) } return int64(len(digests)), combineRowDigests(digests), columns, nil } // orphanIndexDocuments 找出"索引里有、生产分块里没有"的文档(按 chunk_id 关联)。 func (t *ProductionLibraryTarget) orphanIndexDocuments(ctx context.Context, documents []SearchDocument) []string { if len(documents) == 0 { return nil } // 按文档归属判断而不是按 chunk_id:真实 WeKnora 是父子分块, // 索引里的子块 ID 本来就不在 chunks 表里,按 chunk_id 判会误报孤儿。 rows, err := t.pool.Query(ctx, `SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL`, t.scope.TenantID, t.scope.KnowledgeBaseID) if err != nil { return nil } defer rows.Close() known := map[string]bool{} for rows.Next() { var id string if err := rows.Scan(&id); err != nil { return nil } known[id] = true } orphans := make([]string, 0, 4) for _, document := range documents { var source struct { ChunkID string `json:"chunk_id"` KnowledgeID string `json:"knowledge_id"` } if json.Unmarshal(document.Source, &source) != nil || source.KnowledgeID == "" { continue } if !known[source.KnowledgeID] { orphans = append(orphans, document.ID) } } return orphans } // firstIndexDifference 返回第一处不一致的文档 ID 与字段,便于定位。 func firstIndexDifference(archived, live []SearchDocument) string { liveByID := make(map[string]json.RawMessage, len(live)) for _, document := range live { liveByID[document.ID] = document.Source } seen := make(map[string]bool, len(archived)) for _, document := range archived { seen[document.ID] = true current, ok := liveByID[document.ID] if !ok { return document.ID + "(生产缺少)" } var want, got map[string]json.RawMessage if json.Unmarshal(document.Source, &want) != nil || json.Unmarshal(current, &got) != nil { return document.ID + "(无法解析)" } for key, value := range want { other, exists := got[key] if !exists { return fmt.Sprintf("%s(生产缺少字段 %s)", document.ID, key) } if string(value) != string(other) { return fmt.Sprintf("%s(字段 %s 不同:归档 %.60s 生产 %.60s)", document.ID, key, value, other) } } for key := range got { if _, exists := want[key]; !exists { return fmt.Sprintf("%s(生产多出字段 %s)", document.ID, key) } } } for _, document := range live { if !seen[document.ID] { return document.ID + "(生产多余)" } } return "数量一致但摘要不同" } // verifyIndexDigest 比对归档索引文档与生产实际索引文档的规范化摘要。 func (t *ProductionLibraryTarget) verifyIndexDigest(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error { for _, part := range manifest.Parts { if part.Kind != "index" { continue } reader, err := archive.OpenPart(part) if err != nil { return err } documents, err := librarysnapshot.ReadIndexDocuments(reader) reader.Close() if err != nil { return err } archived := make([]SearchDocument, 0, len(documents)) for _, document := range documents { // 写入时 BulK Index 会把 tenant_id/knowledge_base_id 覆盖成本目标的身份, // 校验必须用同样的规范化,否则"写对了"也会被判不一致。 source := map[string]json.RawMessage{} if err := json.Unmarshal(document.Source, &source); err != nil { return errors.New("归档索引文档不是对象") } source["tenant_id"], _ = json.Marshal(t.scope.TenantID) source["knowledge_base_id"], _ = json.Marshal(t.scope.KnowledgeBaseID) encoded, err := json.Marshal(source) if err != nil { return err } canonical, err := canonicalJSON(encoded) if err != nil { return errors.New("归档索引文档无法规范化") } archived = append(archived, SearchDocument{ID: document.ID, Source: canonical}) } wantDigest, err := searchDocumentsDigest(archived) if err != nil { return err } snapshot, err := t.search.ExportLibrary(ctx, SearchIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID}, t.config.MaxDocuments) if err != nil { return fmt.Errorf("读取生产索引失败: %w", err) } // 孤儿向量:索引里有、数据库分块里没有的文档必须报出来。 // 只比"归档 vs 生产"会漏掉两边一致但都错的情况。 if orphans := t.orphanIndexDocuments(ctx, snapshot.Documents); len(orphans) > 0 { return fmt.Errorf("生产索引存在 %d 条孤儿文档(所属文档已不在数据库),首个 %s", len(orphans), orphans[0]) } if len(snapshot.Documents) != len(archived) || snapshot.DocumentsSHA256 != wantDigest { // 摘要不一致时给出第一处差异文档,运维才能定位是"少了/多了/被改了"。 return fmt.Errorf("生产索引与快照不一致:文档 %d/%d,首个差异 %s", len(snapshot.Documents), len(archived), firstIndexDifference(archived, snapshot.Documents)) } } return nil } // productionResourceContent 只读归档里列出的那些资源行:resources 是多知识库 // 共享表,按库重新推导会漏掉"只被历史上某条正文引用过"的行,恢复得一字不差 // 也会被判失败。这里按身份集合取行,再逐行做内容摘要。 func (t *ProductionLibraryTarget) productionResourceContent(ctx context.Context, ids []string, columns map[string]bool) (int64, string, error) { rows, err := t.pool.Query(ctx, `SELECT to_jsonb(r) FROM resources r WHERE r.id = ANY($1::text[]) ORDER BY r.id`, ids) if err != nil { return 0, "", fmt.Errorf("读取生产 resources 内容失败: %w", err) } defer rows.Close() digests := make([]string, 0, len(ids)) for rows.Next() { var raw map[string]json.RawMessage if err := rows.Scan(&raw); err != nil { return 0, "", err } projected := make(map[string]json.RawMessage, len(columns)) for key := range columns { if value, ok := raw[key]; ok { projected[key] = value } } digest, err := canonicalRowDigest(projected) if err != nil { return 0, "", err } digests = append(digests, digest) } if err := rows.Err(); err != nil { return 0, "", err } return int64(len(digests)), combineRowDigests(digests), nil }