5defe238e0
build / build (push) Successful in 1m34s
真实事故:源 KB(10000/0c84a098) 同步到目标 KB(10001/e64e82b5) 时,归档行被原样 插入目标库,落在来源租户/知识库;资源阶段随即报"生产缺少资源 ... 的存储路径" (资源查询按目标租户过滤,看不见刚写进去的行)。回滚只按目标范围回放,于是页面 报"已恢复到预备份",目标却多出 4 文件/38 分块/21 资源/4 绑定。 改: - 写入与校验共用 retargetRow:tenant_id / knowledge_base_id / 资源定位符(后端、 provider、桶、租户前缀) / storage_backend_id / provider / embedding_model_id 统一映射到目标范围;行里的来源身份对不上就失败,宁可不写。 - Verify 增加"本次写集合仍以来源身份残留"判定;Restore 增加整库指纹比对(预备份 时刻 vs 恢复后)与孤儿资源清理,回滚后用 Verify 复核,恢复工具退出 0 不再等于 恢复完成。 - 新增 cmd/libraryresidue:按归档身份精确、可备份、可复核地清理已知的范围外残留。 验证:make verify 全绿;真实 PG+ES 用例复现"源目标 KB 不同 + 目标非空 + 资源阶段 失败",断言回滚后无来源范围残留、其它租户行未动、恢复后内容=预备份;正向用例断言 行落在目标范围且资源定位符指向目标存储。
116 lines
4.8 KiB
Go
116 lines
4.8 KiB
Go
// command libraryresidue 处理"整库同步写到目标范围外"留下的残留行:
|
|
//
|
|
// 读指定快照(来源范围)→ 在目标里找同内容、仍带来源身份的行
|
|
// → 先备份成 0600 文件并记录摘要 → 精确按归档身份删除 → 复核来源身份已空、
|
|
// 目标内容与预备份一致、整库指纹里没有别的租户
|
|
//
|
|
// 它不做范围广删,也不改任何别的东西:判据是"归档里确实有这些行、内容逐字段
|
|
// 一致",任何一行对不上就整体拒绝。默认是干跑(只报告、不删)。
|
|
//
|
|
// 真实事故:跨范围同步把 4 文件/38 分块/21 资源/4 绑定写成来源租户的行,
|
|
// 回滚只按目标范围回放,于是"已恢复"却留下这些行。这个工具的用途就是在
|
|
// 修复后的代码上线前,用可审计的方式清掉它们。
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"knowledge-governance/internal/appconfig"
|
|
"knowledge-governance/internal/libraryrelease"
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
"knowledge-governance/internal/librarystack"
|
|
v071 "knowledge-governance/internal/releaseadapter/v071"
|
|
)
|
|
|
|
func main() {
|
|
scope := flag.String("scope", "internal", "internal|external")
|
|
snapshot := flag.String("snapshot", "", "sealed snapshot whose source scope is the residue (required)")
|
|
backupSnapshot := flag.String("backup-snapshot", "", "prebackup snapshot the target must be restored to (required)")
|
|
expectSource := flag.String("expect-source", "", "expected source scope as <tenant>/<knowledge-base> (required guard against typos)")
|
|
out := flag.String("out", "", "directory for the residue backup (required)")
|
|
apply := flag.Bool("apply", false, "actually delete the residue rows (default: dry run)")
|
|
reportPath := flag.String("report", "", "write the JSON report here")
|
|
flag.Parse()
|
|
if *snapshot == "" || *backupSnapshot == "" || *out == "" || *expectSource == "" {
|
|
log.Fatal("snapshot, backup-snapshot, out and expect-source are required")
|
|
}
|
|
config, err := appconfig.Load(os.Getenv)
|
|
if err != nil {
|
|
log.Fatalf("load config: %v", err)
|
|
}
|
|
ctx := context.Background()
|
|
report, err := run(ctx, config, *scope, *snapshot, *backupSnapshot, *expectSource, *out, *apply)
|
|
if *reportPath != "" && report != nil {
|
|
raw, marshalErr := json.MarshalIndent(report, "", " ")
|
|
if marshalErr == nil {
|
|
raw = append(raw, '\n')
|
|
if writeErr := os.WriteFile(*reportPath, raw, 0o600); writeErr != nil {
|
|
log.Printf("report write failed: %v", writeErr)
|
|
}
|
|
}
|
|
}
|
|
if err != nil {
|
|
log.Fatalf("residue cleanup failed: %v", err)
|
|
}
|
|
log.Printf("residue cleanup ok: source=%s/%s tables=%v cleared=%d applied=%v verified=%v",
|
|
reportSource(report), report.SourceKB, report.Tables, report.Cleared, report.Applied, report.Verified)
|
|
}
|
|
|
|
func reportSource(report *v071.ResidueReport) string {
|
|
return strconv.FormatUint(report.SourceTenant, 10)
|
|
}
|
|
|
|
func run(ctx context.Context, config appconfig.Config, scope, snapshotID, backupSnapshotID, expectSource, out string,
|
|
apply bool) (*v071.ResidueReport, error) {
|
|
if scope != "internal" && scope != "external" {
|
|
return nil, fmt.Errorf("scope must be internal or external")
|
|
}
|
|
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("归档不可用: %w", err)
|
|
}
|
|
manifest, err := archive.Load(snapshotID)
|
|
archive.Close()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("指定快照不可读: %w", err)
|
|
}
|
|
source := v071.ProductionScope{TenantID: manifest.TenantID, KnowledgeBaseID: manifest.KnowledgeBaseID}
|
|
want := strconv.FormatUint(source.TenantID, 10) + "/" + source.KnowledgeBaseID
|
|
// 范围写错会把"别人的行"当成残留:必须由调用方显式确认。
|
|
if strings.TrimSpace(expectSource) != want {
|
|
return nil, fmt.Errorf("来源范围与 -expect-source 不一致:归档为 %s,参数为 %s", want, expectSource)
|
|
}
|
|
target, _, err := librarystack.OpenProductionStack(ctx, config, scope, backupSnapshotID, "")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("装配目标失败: %w", err)
|
|
}
|
|
defer target.Close()
|
|
report, err := target.ClearForeignScope(ctx, source, snapshotID, out, apply)
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
report.Foreign, _ = target.ForeignScopes(ctx)
|
|
if !apply {
|
|
return report, nil
|
|
}
|
|
if err := target.VerifyResidueCleared(ctx, source); err != nil {
|
|
return report, err
|
|
}
|
|
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupSnapshotID}); err != nil {
|
|
return report, fmt.Errorf("清理后目标内容仍与预备份不一致(保持停写): %w", err)
|
|
}
|
|
report.Verified = true
|
|
if len(report.Foreign) > 0 {
|
|
raw, _ := json.Marshal(report.Foreign)
|
|
return report, fmt.Errorf("清理后目标仍有其它租户的数据,需人工确认才能开放:%s", raw)
|
|
}
|
|
return report, nil
|
|
}
|