姚章浩
5defe238e0
build / build (push) Successful in 1m34s
真实事故：源 KB(10000/0c84a098) 同步到目标 KB(10001/e64e82b5) 时，归档行被原样 插入目标库，落在来源租户/知识库；资源阶段随即报"生产缺少资源 ... 的存储路径" （资源查询按目标租户过滤，看不见刚写进去的行）。回滚只按目标范围回放，于是页面 报"已恢复到预备份"，目标却多出 4 文件/38 分块/21 资源/4 绑定。 改： - 写入与校验共用 retargetRow：tenant_id / knowledge_base_id / 资源定位符(后端、 provider、桶、租户前缀) / storage_backend_id / provider / embedding_model_id 统一映射到目标范围；行里的来源身份对不上就失败，宁可不写。 - Verify 增加"本次写集合仍以来源身份残留"判定；Restore 增加整库指纹比对（预备份 时刻 vs 恢复后）与孤儿资源清理，回滚后用 Verify 复核，恢复工具退出 0 不再等于 恢复完成。 - 新增 cmd/libraryresidue：按归档身份精确、可备份、可复核地清理已知的范围外残留。 验证：make verify 全绿；真实 PG+ES 用例复现"源目标 KB 不同 + 目标非空 + 资源阶段 失败"，断言回滚后无来源范围残留、其它租户行未动、恢复后内容=预备份；正向用例断言 行落在目标范围且资源定位符指向目标存储。
335 lines
12 KiB
Go
335 lines
12 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// ResidueReport 是"清理范围外残留"的审计记录：清掉了什么、依据是哪份归档、
|
||
// 备份写到哪儿、清理后目标处于什么状态。没有这份记录就不许动目标。
|
||
type ResidueReport struct {
|
||
SnapshotID string `json:"snapshot_id"`
|
||
SourceTenant uint64 `json:"source_tenant"`
|
||
SourceKB string `json:"source_kb"`
|
||
Tables map[string]int `json:"tables"`
|
||
BackupPath string `json:"backup_path"`
|
||
BackupSHA256 string `json:"backup_sha256"`
|
||
Applied bool `json:"applied"`
|
||
Cleared int `json:"cleared"`
|
||
Verified bool `json:"verified"`
|
||
Foreign map[string]string `json:"foreign_scopes,omitempty"`
|
||
}
|
||
|
||
// ClearForeignScope 清掉"归档里确实有、却仍以来源身份躺在目标里"的行。
|
||
//
|
||
// 它只做这一件事，而且必须全部条件成立才允许删：
|
||
// 1. 目标这些行的内容与指定归档行逐字段一致（先证明它们是这次写入的副本，
|
||
// 不是目标自己的数据）；
|
||
// 2. 先把它们完整导出成 0600 的备份文件并记下摘要；
|
||
// 3. 删除只按"归档身份集合 + 来源租户"精确命中，不做范围广删。
|
||
//
|
||
// 清完还要复核：来源身份不再存在、目标内容与预备份一致、整库指纹里没有别的租户。
|
||
func (t *ProductionLibraryTarget) ClearForeignScope(ctx context.Context, source ProductionScope, snapshotID,
|
||
backupDir string, apply bool) (*ResidueReport, error) {
|
||
if source.TenantID == 0 || !validSearchIdentifier(source.KnowledgeBaseID) {
|
||
return nil, errors.New("来源范围不完整，拒绝清理")
|
||
}
|
||
if source.TenantID == t.scope.TenantID && source.KnowledgeBaseID == t.scope.KnowledgeBaseID {
|
||
return nil, errors.New("来源与目标范围相同，不需要清理")
|
||
}
|
||
if backupDir == "" {
|
||
return nil, errors.New("清理前必须指定备份目录")
|
||
}
|
||
if err := t.resolveStorageBackend(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024)
|
||
if err != nil {
|
||
return nil, errors.New("归档不可用")
|
||
}
|
||
defer archive.Close()
|
||
manifest, err := archive.Load(snapshotID)
|
||
if err != nil {
|
||
return nil, errors.New("指定快照不可读")
|
||
}
|
||
report := &ResidueReport{SnapshotID: snapshotID, SourceTenant: source.TenantID, SourceKB: source.KnowledgeBaseID,
|
||
Tables: map[string]int{}, Applied: false}
|
||
backup := map[string][]map[string]json.RawMessage{}
|
||
digests := map[string]string{}
|
||
identities := map[string][]string{}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "database" && part.Kind != "wiki" {
|
||
continue
|
||
}
|
||
archived, archivedDigests, err := archivedRowsWithDigests(archive, part)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
// 事故里的残留行是"归档行 + 只把桶/后端换成目标"的旧变换结果：判等时必须
|
||
// 同时接受原样与这一种变换，否则会把真残留当成"不是这次写的"而拒绝清理。
|
||
if part.Name == "resources" {
|
||
for _, row := range archived {
|
||
legacy, legacyErr := t.legacyResourceRow(row)
|
||
if legacyErr != nil {
|
||
return nil, legacyErr
|
||
}
|
||
digest, digestErr := canonicalRowDigest(legacy)
|
||
if digestErr != nil {
|
||
return nil, digestErr
|
||
}
|
||
archivedDigests[digest] = true
|
||
}
|
||
}
|
||
if len(archived) == 0 {
|
||
continue
|
||
}
|
||
live, err := t.foreignScopeRows(ctx, part.Name, source)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
liveDigests := make(map[string]string, len(live))
|
||
for _, row := range live {
|
||
digest, err := canonicalRowDigest(row)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
liveDigests[digest] = archiveRowIdentity(part.Name, row)
|
||
}
|
||
// 目标里这一范围的行必须都能在归档里找到同样内容：否则它们不是这次写入的
|
||
// 副本，删了就是删别人的数据。
|
||
rows := make([]map[string]json.RawMessage, 0, len(live))
|
||
ids := make([]string, 0, len(live))
|
||
for digest, identity := range liveDigests {
|
||
if _, ok := archivedDigests[digest]; !ok {
|
||
return nil, fmt.Errorf("目标 %s 的行 %s 与指定归档不一致，拒绝按本次写入清理", part.Name, identity)
|
||
}
|
||
ids = append(ids, identity)
|
||
}
|
||
for _, row := range live {
|
||
rows = append(rows, row)
|
||
}
|
||
sort.Strings(ids)
|
||
report.Tables[part.Name] = len(rows)
|
||
backup[part.Name] = rows
|
||
identities[part.Name] = ids
|
||
digests[part.Name] = combineRowDigests(sortedKeys(liveDigests))
|
||
}
|
||
total := 0
|
||
for _, count := range report.Tables {
|
||
total += count
|
||
}
|
||
if total == 0 {
|
||
report.Verified = true
|
||
return report, nil
|
||
}
|
||
if err := os.MkdirAll(backupDir, 0o700); err != nil {
|
||
return nil, fmt.Errorf("创建备份目录失败: %w", err)
|
||
}
|
||
payload, err := json.MarshalIndent(map[string]any{
|
||
"snapshot_id": snapshotID, "source_tenant": source.TenantID, "source_kb": source.KnowledgeBaseID,
|
||
"target_tenant": t.scope.TenantID, "target_kb": t.scope.KnowledgeBaseID,
|
||
"tables": report.Tables, "rows": backup, "digests": digests,
|
||
}, "", " ")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
report.BackupPath = filepath.Join(backupDir, "foreign-scope-residue-"+snapshotID[:12]+".json")
|
||
if err := os.WriteFile(report.BackupPath, payload, 0o600); err != nil {
|
||
return nil, fmt.Errorf("写残留备份失败: %w", err)
|
||
}
|
||
sum := sha256.Sum256(payload)
|
||
report.BackupSHA256 = hex.EncodeToString(sum[:])
|
||
if !apply {
|
||
return report, nil
|
||
}
|
||
// 删除顺序：子表先、父表后（resource_bindings → resources，chunks → knowledges）。
|
||
for index := len(productionLibraryTables) - 1; index >= 0; index-- {
|
||
table := productionLibraryTables[index]
|
||
ids, ok := identities[table]
|
||
if !ok || len(ids) == 0 {
|
||
continue
|
||
}
|
||
statement := fmt.Sprintf(`DELETE FROM %s WHERE id = ANY($1::text[])`, table)
|
||
if !tableHasTenant(table) {
|
||
// knowledge_tag_relations 没有 id：身份就是父文档 + 标签。
|
||
statement = `DELETE FROM knowledge_tag_relations WHERE (knowledge_id||':'||tag_id) = ANY($1::text[])`
|
||
}
|
||
if _, err := t.pool.Exec(ctx, statement, ids); err != nil {
|
||
return nil, fmt.Errorf("清理 %s 失败: %w", table, err)
|
||
}
|
||
}
|
||
report.Applied = true
|
||
report.Cleared = total
|
||
return report, nil
|
||
}
|
||
|
||
// legacyResourceRow 复现旧实现写资源行时的变换：只换定位符里的存储后端与桶，
|
||
// 租户/知识库/provider 原样保留。判等要认这一种形状，事故残留才是"可证明"的。
|
||
func (t *ProductionLibraryTarget) legacyResourceRow(row map[string]json.RawMessage) (map[string]json.RawMessage, error) {
|
||
copied := make(map[string]json.RawMessage, len(row))
|
||
for key, value := range row {
|
||
copied[key] = value
|
||
}
|
||
if t.config.ObjectsBucket == "" {
|
||
return copied, nil
|
||
}
|
||
raw, ok := copied["physical_path"]
|
||
if !ok {
|
||
return nil, errors.New("资源行缺少定位符")
|
||
}
|
||
var locator string
|
||
if err := json.Unmarshal(raw, &locator); err != nil {
|
||
return nil, errors.New("资源定位符不可解析")
|
||
}
|
||
if rewritten, ok := rewriteLocatorTarget(locator, t.storageBackend, t.config.ObjectsBucket); ok {
|
||
encoded, err := json.Marshal(rewritten)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
copied["physical_path"] = encoded
|
||
}
|
||
if t.storageBackend == "" {
|
||
return copied, nil
|
||
}
|
||
if raw, ok := copied["storage_backend_id"]; ok {
|
||
var current string
|
||
if json.Unmarshal(raw, ¤t) == nil && current != "" && current != t.storageBackend {
|
||
encoded, err := json.Marshal(t.storageBackend)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
copied["storage_backend_id"] = encoded
|
||
}
|
||
}
|
||
return copied, nil
|
||
}
|
||
|
||
// VerifyResidueCleared 复核清理结果：来源身份不再存在、其它租户的解释得清、
|
||
// 且（可选）目标内容与预备份完全一致。
|
||
func (t *ProductionLibraryTarget) VerifyResidueCleared(ctx context.Context, source ProductionScope) error {
|
||
for _, table := range productionLibraryTables {
|
||
if !tableHasTenant(table) {
|
||
continue
|
||
}
|
||
statement := fmt.Sprintf(`SELECT count(*) FROM %s WHERE tenant_id=$1`, table)
|
||
arguments := []any{source.TenantID}
|
||
if tablesWithKB[table] {
|
||
statement += " AND knowledge_base_id=$2"
|
||
arguments = append(arguments, source.KnowledgeBaseID)
|
||
}
|
||
var count int64
|
||
if err := t.pool.QueryRow(ctx, statement, arguments...).Scan(&count); err != nil {
|
||
return fmt.Errorf("复核 %s 失败: %w", table, err)
|
||
}
|
||
if count > 0 {
|
||
return fmt.Errorf("目标 %s 仍有 %d 行属于来源范围 %d/%s", table, count, source.TenantID, source.KnowledgeBaseID)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ForeignScopes 列出目标里所有"不是目标租户"的租户，用于人工确认没有别的泄漏。
|
||
func (t *ProductionLibraryTarget) ForeignScopes(ctx context.Context) (map[string]string, error) {
|
||
foreign := map[string]string{}
|
||
for _, table := range productionLibraryTables {
|
||
if !tableHasTenant(table) {
|
||
continue
|
||
}
|
||
rows, err := t.pool.Query(ctx, fmt.Sprintf(
|
||
`SELECT tenant_id, count(*) FROM %s WHERE tenant_id <> $1 GROUP BY 1`, table), t.scope.TenantID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取 %s 租户分布失败: %w", table, err)
|
||
}
|
||
for rows.Next() {
|
||
var tenant, count int64
|
||
if err := rows.Scan(&tenant, &count); err != nil {
|
||
rows.Close()
|
||
return nil, err
|
||
}
|
||
foreign[fmt.Sprintf("%s:%d", table, tenant)] = fmt.Sprintf("%d 行", count)
|
||
}
|
||
rows.Close()
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
return foreign, nil
|
||
}
|
||
|
||
// foreignScopeRows 读出目标里属于来源范围的行（resources / resource_bindings
|
||
// 只有租户列，按租户取；其余按租户+知识库取）。
|
||
func (t *ProductionLibraryTarget) foreignScopeRows(ctx context.Context, table string, source ProductionScope) (
|
||
[]map[string]json.RawMessage, error) {
|
||
if !tableHasTenant(table) {
|
||
return nil, nil
|
||
}
|
||
statement := fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table))
|
||
arguments := []any{source.TenantID, source.KnowledgeBaseID}
|
||
if table == "resources" || table == "resource_bindings" {
|
||
statement = fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE tenant_id=$1`, table, table)
|
||
arguments = arguments[:1]
|
||
}
|
||
rows, err := t.pool.Query(ctx, statement, arguments...)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取目标 %s 来源范围行失败: %w", table, err)
|
||
}
|
||
defer rows.Close()
|
||
result := make([]map[string]json.RawMessage, 0, 8)
|
||
for rows.Next() {
|
||
var row map[string]json.RawMessage
|
||
if err := rows.Scan(&row); err != nil {
|
||
return nil, err
|
||
}
|
||
result = append(result, row)
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
// archivedRowsWithDigests 读出一份归档部件的行与规范化内容摘要。
|
||
func archivedRowsWithDigests(archive *librarysnapshot.Archive, part librarysnapshot.Part) (
|
||
[]map[string]json.RawMessage, map[string]bool, error) {
|
||
reader, err := archive.OpenPart(part)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
defer reader.Close()
|
||
decoder := json.NewDecoder(reader)
|
||
token, err := decoder.Token()
|
||
if err != nil || token != json.Delim('[') {
|
||
return nil, nil, errors.New("归档部件不是行数组")
|
||
}
|
||
rows := make([]map[string]json.RawMessage, 0, 64)
|
||
digests := map[string]bool{}
|
||
for decoder.More() {
|
||
var row map[string]json.RawMessage
|
||
if err := decoder.Decode(&row); err != nil {
|
||
return nil, nil, err
|
||
}
|
||
digest, err := canonicalRowDigest(row)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
digests[digest] = true
|
||
rows = append(rows, row)
|
||
}
|
||
return rows, digests, nil
|
||
}
|
||
|
||
func sortedKeys(values map[string]string) []string {
|
||
keys := make([]string, 0, len(values))
|
||
for key := range values {
|
||
keys = append(keys, key)
|
||
}
|
||
sort.Strings(keys)
|
||
return keys
|
||
}
|