姚章浩
5defe238e0
build / build (push) Successful in 1m34s
真实事故：源 KB(10000/0c84a098) 同步到目标 KB(10001/e64e82b5) 时，归档行被原样 插入目标库，落在来源租户/知识库；资源阶段随即报"生产缺少资源 ... 的存储路径" （资源查询按目标租户过滤，看不见刚写进去的行）。回滚只按目标范围回放，于是页面 报"已恢复到预备份"，目标却多出 4 文件/38 分块/21 资源/4 绑定。 改： - 写入与校验共用 retargetRow：tenant_id / knowledge_base_id / 资源定位符(后端、 provider、桶、租户前缀) / storage_backend_id / provider / embedding_model_id 统一映射到目标范围；行里的来源身份对不上就失败，宁可不写。 - Verify 增加"本次写集合仍以来源身份残留"判定；Restore 增加整库指纹比对（预备份 时刻 vs 恢复后）与孤儿资源清理，回滚后用 Verify 复核，恢复工具退出 0 不再等于 恢复完成。 - 新增 cmd/libraryresidue：按归档身份精确、可备份、可复核地清理已知的范围外残留。 验证：make verify 全绿；真实 PG+ES 用例复现"源目标 KB 不同 + 目标非空 + 资源阶段 失败"，断言回滚后无来源范围残留、其它租户行未动、恢复后内容=预备份；正向用例断言 行落在目标范围且资源定位符指向目标存储。
195 lines
7.7 KiB
Go
195 lines
7.7 KiB
Go
package v071
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// 真实事故的形状：来源 KB 与目标 KB 不同（租户也不同），归档行原样写进目标，
|
|
// 目标范围的删除/校验/恢复都看不到它们。这里把"必须改写哪些列"钉死。
|
|
|
|
func scopeTargetFixture() *ProductionLibraryTarget {
|
|
return &ProductionLibraryTarget{
|
|
scope: ProductionScope{TenantID: 10001, KnowledgeBaseID: "kb-target"},
|
|
config: ProductionLibraryConfig{ObjectsBucket: "kgseal-1029a"},
|
|
storageBackend: "backend-target",
|
|
storageProvider: "minio",
|
|
snapshotTenant: 10000,
|
|
snapshotKB: "kb-source",
|
|
}
|
|
}
|
|
|
|
func rawRow(t *testing.T, fields map[string]any) map[string]json.RawMessage {
|
|
t.Helper()
|
|
row := make(map[string]json.RawMessage, len(fields))
|
|
for key, value := range fields {
|
|
encoded, err := json.Marshal(value)
|
|
if err != nil {
|
|
t.Fatalf("encode %s: %v", key, err)
|
|
}
|
|
row[key] = encoded
|
|
}
|
|
return row
|
|
}
|
|
|
|
func fieldText(t *testing.T, row map[string]json.RawMessage, key string) string {
|
|
t.Helper()
|
|
var value string
|
|
if err := json.Unmarshal(row[key], &value); err != nil {
|
|
t.Fatalf("%s not a string: %v", key, err)
|
|
}
|
|
return value
|
|
}
|
|
|
|
func fieldUint(t *testing.T, row map[string]json.RawMessage, key string) uint64 {
|
|
t.Helper()
|
|
var value uint64
|
|
if err := json.Unmarshal(row[key], &value); err != nil {
|
|
t.Fatalf("%s not a number: %v", key, err)
|
|
}
|
|
return value
|
|
}
|
|
|
|
func TestRetargetRowMapsSourceScopeToTarget(t *testing.T) {
|
|
target := scopeTargetFixture()
|
|
for _, table := range []string{"knowledges", "chunks", "knowledge_tags", "wiki_pages", "wiki_folders"} {
|
|
row := rawRow(t, map[string]any{"id": "row-1", "tenant_id": 10000, "knowledge_base_id": "kb-source"})
|
|
if err := target.retargetRow(table, row); err != nil {
|
|
t.Fatalf("%s retarget: %v", table, err)
|
|
}
|
|
if got := fieldUint(t, row, "tenant_id"); got != 10001 {
|
|
t.Fatalf("%s tenant_id = %d, want 10001", table, got)
|
|
}
|
|
if got := fieldText(t, row, "knowledge_base_id"); got != "kb-target" {
|
|
t.Fatalf("%s knowledge_base_id = %s, want kb-target", table, got)
|
|
}
|
|
}
|
|
// 共享表只有租户列：不得凭空写一个 knowledge_base_id 出来。
|
|
binding := rawRow(t, map[string]any{"id": "bind-1", "tenant_id": 10000, "owner_type": "knowledge", "owner_id": "row-1"})
|
|
if err := target.retargetRow("resource_bindings", binding); err != nil {
|
|
t.Fatalf("resource_bindings retarget: %v", err)
|
|
}
|
|
if got := fieldUint(t, binding, "tenant_id"); got != 10001 {
|
|
t.Fatalf("binding tenant_id = %d, want 10001", got)
|
|
}
|
|
if _, ok := binding["knowledge_base_id"]; ok {
|
|
t.Fatal("resource_bindings must not gain a knowledge_base_id column")
|
|
}
|
|
// knowledge_tag_relations 没有范围列：归属走父表，改写必须是空操作。
|
|
relation := rawRow(t, map[string]any{"knowledge_id": "row-1", "tag_id": "tag-1"})
|
|
if err := target.retargetRow("knowledge_tag_relations", relation); err != nil {
|
|
t.Fatalf("knowledge_tag_relations retarget: %v", err)
|
|
}
|
|
if len(relation) != 2 {
|
|
t.Fatalf("knowledge_tag_relations must stay untouched, got %v", relation)
|
|
}
|
|
}
|
|
|
|
// 文档行的 embedding 模型必须换成目标知识库绑定的那一个：来源模型 id 在目标
|
|
// 实例里通常不存在，留着它检索时的查询向量与库里向量就不是同一套。
|
|
func TestRetargetRowUsesTargetEmbeddingModel(t *testing.T) {
|
|
target := scopeTargetFixture()
|
|
target.targetEmbeddingModel = "model-target"
|
|
row := rawRow(t, map[string]any{"id": "doc-1", "tenant_id": 10000, "knowledge_base_id": "kb-source",
|
|
"embedding_model_id": "model-source"})
|
|
if err := target.retargetRow("knowledges", row); err != nil {
|
|
t.Fatalf("retarget: %v", err)
|
|
}
|
|
if got := fieldText(t, row, "embedding_model_id"); got != "model-target" {
|
|
t.Fatalf("embedding_model_id = %s, want model-target", got)
|
|
}
|
|
// 目标没绑模型时不猜：保持来源值，交给契约校验去发现不一致。
|
|
plain := scopeTargetFixture()
|
|
row = rawRow(t, map[string]any{"id": "doc-1", "tenant_id": 10000, "knowledge_base_id": "kb-source",
|
|
"embedding_model_id": "model-source"})
|
|
if err := plain.retargetRow("knowledges", row); err != nil {
|
|
t.Fatalf("retarget without target model: %v", err)
|
|
}
|
|
if got := fieldText(t, row, "embedding_model_id"); got != "model-source" {
|
|
t.Fatalf("embedding_model_id = %s, want unchanged", got)
|
|
}
|
|
}
|
|
|
|
func TestRetargetRowFailsClosedOutsideSnapshotScope(t *testing.T) {
|
|
target := scopeTargetFixture()
|
|
cases := map[string]map[string]any{
|
|
"foreign tenant": {"id": "x", "tenant_id": 20000, "knowledge_base_id": "kb-source"},
|
|
"foreign knowledge": {"id": "x", "tenant_id": 10000, "knowledge_base_id": "kb-other"},
|
|
"missing tenant": {"id": "x", "knowledge_base_id": "kb-source"},
|
|
"missing knowledge kb": {"id": "x", "tenant_id": 10000},
|
|
}
|
|
for name, fields := range cases {
|
|
if err := target.retargetRow("knowledges", rawRow(t, fields)); err == nil {
|
|
t.Fatalf("%s: retarget must fail closed", name)
|
|
}
|
|
}
|
|
// 快照来源未知时不许猜目标范围。
|
|
unknown := &ProductionLibraryTarget{scope: ProductionScope{TenantID: 10001, KnowledgeBaseID: "kb-target"}}
|
|
if err := unknown.retargetRow("knowledges", rawRow(t, map[string]any{"id": "x", "tenant_id": 10000, "knowledge_base_id": "kb"})); err == nil {
|
|
t.Fatal("unknown snapshot scope must fail closed")
|
|
}
|
|
}
|
|
|
|
func TestRetargetRowRewritesResourceStorageIdentity(t *testing.T) {
|
|
target := scopeTargetFixture()
|
|
row := rawRow(t, map[string]any{
|
|
"id": "res-1",
|
|
"tenant_id": 10000,
|
|
"storage_backend_id": "backend-source",
|
|
"provider": "oss",
|
|
"physical_path": "storage://backend-source/oss://yzh-weknora-test/weknora/10000/res-1/a.png",
|
|
})
|
|
if err := target.retargetRow("resources", row); err != nil {
|
|
t.Fatalf("retarget resource: %v", err)
|
|
}
|
|
if got := fieldUint(t, row, "tenant_id"); got != 10001 {
|
|
t.Fatalf("resource tenant = %d, want 10001", got)
|
|
}
|
|
want := "storage://backend-target/minio://kgseal-1029a/10001/res-1/a.png"
|
|
if got := fieldText(t, row, "physical_path"); got != want {
|
|
t.Fatalf("physical_path = %q, want %q", got, want)
|
|
}
|
|
if got := fieldText(t, row, "provider"); got != "minio" {
|
|
t.Fatalf("provider = %q, want minio", got)
|
|
}
|
|
if got := fieldText(t, row, "storage_backend_id"); got != "backend-target" {
|
|
t.Fatalf("storage_backend_id = %q, want backend-target", got)
|
|
}
|
|
}
|
|
|
|
func TestRetargetKeyTenantDropsSourcePrefix(t *testing.T) {
|
|
cases := []struct{ key, want string }{
|
|
{"weknora/10000/exports/a.png", "10001/exports/a.png"},
|
|
{"10000/knowledge/file.md", "10001/knowledge/file.md"},
|
|
{"other/prefix/a.png", "other/prefix/a.png"},
|
|
{"10000", "10000"},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := retargetKeyTenant(tc.key, 10000, 10001); got != tc.want {
|
|
t.Fatalf("retargetKeyTenant(%q) = %q, want %q", tc.key, got, tc.want)
|
|
}
|
|
}
|
|
// 同租户不编码：键必须原样返回。
|
|
if got := retargetKeyTenant("weknora/10000/a.png", 10000, 10000); got != "weknora/10000/a.png" {
|
|
t.Fatalf("same tenant must keep key, got %q", got)
|
|
}
|
|
}
|
|
|
|
func TestRetargetLocatorRejectsMalformed(t *testing.T) {
|
|
if _, err := retargetLocator(" not-a-locator ", "backend", "minio", "bucket", 10000, 10001); err == nil {
|
|
t.Fatal("locator with surrounding spaces must be rejected")
|
|
}
|
|
if _, err := retargetLocator("storage://backend/oss://", "backend", "minio", "bucket", 10000, 10001); err == nil {
|
|
t.Fatal("locator without object key must be rejected")
|
|
}
|
|
// 没有目标桶（测试夹具）时不改写，保持原值可继续用于采集/校验。
|
|
locator := "storage://backend/oss://bucket/key"
|
|
if got, err := retargetLocator(locator, "backend", "minio", "", 10000, 10001); err != nil || got != locator {
|
|
t.Fatalf("empty bucket must be a no-op, got %q err=%v", got, err)
|
|
}
|
|
if !strings.Contains(retargetKeyTenant("weknora/10000/a", 10000, 10001), "10001") {
|
|
t.Fatal("tenant segment must be replaced")
|
|
}
|
|
}
|