Files
knowledge-governance/cmd/libraryresidue/main.go
T
姚章浩 5defe238e0
build / build (push) Successful in 1m34s
fix(library): 跨范围同步按目标范围改写并校验残留
真实事故:源 KB(10000/0c84a098) 同步到目标 KB(10001/e64e82b5) 时,归档行被原样
插入目标库,落在来源租户/知识库;资源阶段随即报"生产缺少资源 ... 的存储路径"
(资源查询按目标租户过滤,看不见刚写进去的行)。回滚只按目标范围回放,于是页面
报"已恢复到预备份",目标却多出 4 文件/38 分块/21 资源/4 绑定。

改:
- 写入与校验共用 retargetRow:tenant_id / knowledge_base_id / 资源定位符(后端、
  provider、桶、租户前缀) / storage_backend_id / provider / embedding_model_id
  统一映射到目标范围;行里的来源身份对不上就失败,宁可不写。
- Verify 增加"本次写集合仍以来源身份残留"判定;Restore 增加整库指纹比对(预备份
  时刻 vs 恢复后)与孤儿资源清理,回滚后用 Verify 复核,恢复工具退出 0 不再等于
  恢复完成。
- 新增 cmd/libraryresidue:按归档身份精确、可备份、可复核地清理已知的范围外残留。

验证:make verify 全绿;真实 PG+ES 用例复现"源目标 KB 不同 + 目标非空 + 资源阶段
失败",断言回滚后无来源范围残留、其它租户行未动、恢复后内容=预备份;正向用例断言
行落在目标范围且资源定位符指向目标存储。
2026-10-29 22:11:59 +08:00

116 lines
4.8 KiB
Go

// command libraryresidue 处理"整库同步写到目标范围外"留下的残留行:
//
// 读指定快照(来源范围)→ 在目标里找同内容、仍带来源身份的行
// → 先备份成 0600 文件并记录摘要 → 精确按归档身份删除 → 复核来源身份已空、
// 目标内容与预备份一致、整库指纹里没有别的租户
//
// 它不做范围广删,也不改任何别的东西:判据是"归档里确实有这些行、内容逐字段
// 一致",任何一行对不上就整体拒绝。默认是干跑(只报告、不删)。
//
// 真实事故:跨范围同步把 4 文件/38 分块/21 资源/4 绑定写成来源租户的行,
// 回滚只按目标范围回放,于是"已恢复"却留下这些行。这个工具的用途就是在
// 修复后的代码上线前,用可审计的方式清掉它们。
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"strconv"
"strings"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
"knowledge-governance/internal/librarystack"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
func main() {
scope := flag.String("scope", "internal", "internal|external")
snapshot := flag.String("snapshot", "", "sealed snapshot whose source scope is the residue (required)")
backupSnapshot := flag.String("backup-snapshot", "", "prebackup snapshot the target must be restored to (required)")
expectSource := flag.String("expect-source", "", "expected source scope as <tenant>/<knowledge-base> (required guard against typos)")
out := flag.String("out", "", "directory for the residue backup (required)")
apply := flag.Bool("apply", false, "actually delete the residue rows (default: dry run)")
reportPath := flag.String("report", "", "write the JSON report here")
flag.Parse()
if *snapshot == "" || *backupSnapshot == "" || *out == "" || *expectSource == "" {
log.Fatal("snapshot, backup-snapshot, out and expect-source are required")
}
config, err := appconfig.Load(os.Getenv)
if err != nil {
log.Fatalf("load config: %v", err)
}
ctx := context.Background()
report, err := run(ctx, config, *scope, *snapshot, *backupSnapshot, *expectSource, *out, *apply)
if *reportPath != "" && report != nil {
raw, marshalErr := json.MarshalIndent(report, "", " ")
if marshalErr == nil {
raw = append(raw, '\n')
if writeErr := os.WriteFile(*reportPath, raw, 0o600); writeErr != nil {
log.Printf("report write failed: %v", writeErr)
}
}
}
if err != nil {
log.Fatalf("residue cleanup failed: %v", err)
}
log.Printf("residue cleanup ok: source=%s/%s tables=%v cleared=%d applied=%v verified=%v",
reportSource(report), report.SourceKB, report.Tables, report.Cleared, report.Applied, report.Verified)
}
func reportSource(report *v071.ResidueReport) string {
return strconv.FormatUint(report.SourceTenant, 10)
}
func run(ctx context.Context, config appconfig.Config, scope, snapshotID, backupSnapshotID, expectSource, out string,
apply bool) (*v071.ResidueReport, error) {
if scope != "internal" && scope != "external" {
return nil, fmt.Errorf("scope must be internal or external")
}
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
if err != nil {
return nil, fmt.Errorf("归档不可用: %w", err)
}
manifest, err := archive.Load(snapshotID)
archive.Close()
if err != nil {
return nil, fmt.Errorf("指定快照不可读: %w", err)
}
source := v071.ProductionScope{TenantID: manifest.TenantID, KnowledgeBaseID: manifest.KnowledgeBaseID}
want := strconv.FormatUint(source.TenantID, 10) + "/" + source.KnowledgeBaseID
// 范围写错会把"别人的行"当成残留:必须由调用方显式确认。
if strings.TrimSpace(expectSource) != want {
return nil, fmt.Errorf("来源范围与 -expect-source 不一致:归档为 %s,参数为 %s", want, expectSource)
}
target, _, err := librarystack.OpenProductionStack(ctx, config, scope, backupSnapshotID, "")
if err != nil {
return nil, fmt.Errorf("装配目标失败: %w", err)
}
defer target.Close()
report, err := target.ClearForeignScope(ctx, source, snapshotID, out, apply)
if err != nil {
return report, err
}
report.Foreign, _ = target.ForeignScopes(ctx)
if !apply {
return report, nil
}
if err := target.VerifyResidueCleared(ctx, source); err != nil {
return report, err
}
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupSnapshotID}); err != nil {
return report, fmt.Errorf("清理后目标内容仍与预备份不一致(保持停写): %w", err)
}
report.Verified = true
if len(report.Foreign) > 0 {
raw, _ := json.Marshal(report.Foreign)
return report, fmt.Errorf("清理后目标仍有其它租户的数据,需人工确认才能开放:%s", raw)
}
return report, nil
}