// library-e2e-seed 从真实归档派生一份内部自洽的"第二版"快照,供候选验收跑更新/撤销。 // 不模拟:所有部件都经 Archive.Put 真实重算哈希并 Seal,产物是合法归档。 package main import ( "bytes" "encoding/json" "flag" "fmt" "io" "log" "regexp" "time" "knowledge-governance/internal/librarysnapshot" ) const maxBlob = 512 << 20 var handlePattern = regexp.MustCompile(`resource://([A-Za-z0-9_-]{4,64})`) type row = map[string]json.RawMessage func main() { root := flag.String("root", "", "归档根") base := flag.String("base", "", "基线快照 ID") mode := flag.String("mode", "update", "update|revoke") flag.Parse() if *root == "" || *base == "" { log.Fatal("root and base are required") } archive, err := librarysnapshot.Open(*root, maxBlob) if err != nil { log.Fatal(err) } defer archive.Close() manifest, err := archive.Load(*base) if err != nil { log.Fatal(err) } // 先把需要的表读进内存:派生规则要跨表看引用关系,不能边读边改。 rows := map[string][]row{} for _, part := range manifest.Parts { if part.Kind != "database" { continue } reader, err := archive.OpenPart(part) if err != nil { log.Fatal(err) } data, err := io.ReadAll(reader) reader.Close() if err != nil { log.Fatal(err) } var parsed []row if err := json.Unmarshal(data, &parsed); err != nil { log.Fatal(err) } rows[part.Name] = parsed } dropped := "" // 撤销文档对应的分块 ID:索引文档一般只有 chunk_id,靠它才能把索引一起撤掉。 droppedChunks := map[string]bool{} if *mode == "revoke" { dropped = smallestID(rows["knowledges"]) for _, item := range rows["chunks"] { if field(item, "knowledge_id") == dropped { droppedChunks[field(item, "id")] = true } } } // 仍被引用的 handle:被撤销文档的分块与本体都不算引用。 referenced := map[string]bool{} for table, items := range rows { if table != "chunks" && table != "knowledges" { continue } for _, item := range items { if *mode == "revoke" && (field(item, "id") == dropped || field(item, "knowledge_id") == dropped) { continue } for _, key := range []string{"content", "description", "title"} { for _, handle := range extractHandles(field(item, key)) { referenced[handle] = true } } } } // 资源部件名是资源 ID:只有当它的 handle 仍被引用时才保留。 // 否则撤销后归档会带一个生产作用域算不到的孤儿资源,校验必然失败。 // 绑定关系同样是"活着"的依据:有些资源只在绑定里出现,正文并不引用。 bound := map[string]bool{} for _, item := range rows["resource_bindings"] { if owner := field(item, "owner_id"); owner == dropped { continue } if resource := field(item, "resource_id"); resource != "" { bound[resource] = true } } keepResource := map[string]bool{} if *mode == "revoke" { for _, item := range rows["resources"] { id := field(item, "id") if referenced[field(item, "handle")] || bound[id] { keepResource[id] = true } } } parts := make([]librarysnapshot.Part, 0, len(manifest.Parts)) for _, part := range manifest.Parts { reader, err := archive.OpenPart(part) if err != nil { log.Fatal(err) } data, err := io.ReadAll(reader) reader.Close() if err != nil { log.Fatal(err) } switch { case *mode == "revoke" && part.Kind == "resource" && !keepResource[part.Name]: continue case part.Kind == "database" && part.Name == "knowledges": data = mutateKnowledges(data, *mode, dropped) case *mode == "revoke" && part.Kind == "database" && part.Name == "resources": // 孤儿资源的行也要一起删:归档要求"每行资源都有对象部件", // 只删部件会得到一份自相矛盾的快照。 data = filterResources(data, keepResource) case *mode == "revoke" && part.Kind == "database" && part.Name == "chunks": data = dropByKnowledge(data, dropped) case *mode == "revoke" && part.Kind == "database" && part.Name == "resource_bindings": data = dropByOwner(data, dropped) case *mode == "revoke" && part.Kind == "index": data = dropIndexRows(data, dropped, droppedChunks) } sealed, err := archive.Put(bytes.NewReader(data)) if err != nil { log.Fatal(err) } sealed.Kind, sealed.Name, sealed.Count = part.Kind, part.Name, part.Count if part.Kind == "database" || part.Kind == "index" { count, countErr := rowCount(data) if countErr != nil { log.Fatal(countErr) } sealed.Count = count } parts = append(parts, sealed) } manifest.Parts = parts manifest.CreatedAt = time.Now().UTC() id, err := archive.Seal(manifest) if err != nil { log.Fatal(err) } fmt.Printf(`{"snapshot_id":%q,"mode":%q,"base":%q,"dropped":%q,"parts":%d}`, id, *mode, *base, dropped, len(parts)) } func extractHandles(text string) []string { matches := handlePattern.FindAllStringSubmatch(text, -1) handles := make([]string, 0, len(matches)) for _, match := range matches { handles = append(handles, match[1]) } return handles } func smallestID(items []row) string { smallest := "" for _, item := range items { id := field(item, "id") if id != "" && (smallest == "" || id < smallest) { smallest = id } } return smallest } func field(item row, key string) string { raw, ok := item[key] if !ok { return "" } var value string _ = json.Unmarshal(raw, &value) return value } // mutateKnowledges:update 改一条标题(内容变更),revoke 删掉指定文档。 func mutateKnowledges(data []byte, mode, dropped string) []byte { var items []row if err := json.Unmarshal(data, &items); err != nil { log.Fatal(err) } target := smallestID(items) for _, item := range items { if mode == "update" && field(item, "id") == target { title, _ := json.Marshal(field(item, "title") + "(E2E 更新)") item["title"] = title } } if mode == "revoke" { kept := make([]row, 0, len(items)) for _, item := range items { if field(item, "id") == dropped { continue } kept = append(kept, item) } return mustJSON(kept) } return mustJSON(items) } // filterResources 只保留仍然被引用的资源行。 func filterResources(data []byte, keep map[string]bool) []byte { var items []row if err := json.Unmarshal(data, &items); err != nil { log.Fatal(err) } kept := make([]row, 0, len(items)) for _, item := range items { if keep[field(item, "id")] { kept = append(kept, item) } } return mustJSON(kept) } // dropIndexRows 既按 knowledge_id 也按 chunk_id 删索引文档: // 真实索引文档通常只带 chunk_id,漏了它就会留下"孤儿向量"。 func dropIndexRows(data []byte, knowledgeID string, chunkIDs map[string]bool) []byte { var items []row if err := json.Unmarshal(data, &items); err != nil { log.Fatal(err) } kept := make([]row, 0, len(items)) for _, item := range items { // 归档索引行是 {ID, Source} 包装;真正字段在 Source 里。 source := item if raw, ok := item["Source"]; ok { var inner row if json.Unmarshal(raw, &inner) == nil { source = inner } } if field(source, "knowledge_id") == knowledgeID { continue } if chunkIDs[field(source, "chunk_id")] || chunkIDs[field(source, "id")] { continue } kept = append(kept, item) } return mustJSON(kept) } func dropByKnowledge(data []byte, knowledgeID string) []byte { var items []row if err := json.Unmarshal(data, &items); err != nil { log.Fatal(err) } kept := make([]row, 0, len(items)) for _, item := range items { if field(item, "knowledge_id") == knowledgeID { continue } kept = append(kept, item) } return mustJSON(kept) } func dropByOwner(data []byte, ownerID string) []byte { var items []row if err := json.Unmarshal(data, &items); err != nil { log.Fatal(err) } kept := make([]row, 0, len(items)) for _, item := range items { if field(item, "owner_id") == ownerID { continue } kept = append(kept, item) } return mustJSON(kept) } func rowCount(data []byte) (int64, error) { var items []json.RawMessage if err := json.Unmarshal(data, &items); err != nil { return 0, err } return int64(len(items)), nil } func mustJSON(value any) []byte { data, err := json.Marshal(value) if err != nil { panic(err) } return data }