knowledge-governance/cmd/library-e2e-seed/main.go

// library-e2e-seed 从真实归档派生一份内部自洽的"第二版"快照，供候选验收跑更新/撤销。
// 不模拟：所有部件都经 Archive.Put 真实重算哈希并 Seal，产物是合法归档。
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"regexp"
	"time"

	"knowledge-governance/internal/librarysnapshot"
)

const maxBlob = 512 << 20

var handlePattern = regexp.MustCompile(`resource://([A-Za-z0-9_-]{4,64})`)

type row = map[string]json.RawMessage

func main() {
	root := flag.String("root", "", "归档根")
	base := flag.String("base", "", "基线快照 ID")
	mode := flag.String("mode", "update", "update|revoke")
	flag.Parse()
	if *root == "" || *base == "" {
		log.Fatal("root and base are required")
	}
	archive, err := librarysnapshot.Open(*root, maxBlob)
	if err != nil {
		log.Fatal(err)
	}
	defer archive.Close()
	manifest, err := archive.Load(*base)
	if err != nil {
		log.Fatal(err)
	}

	// 先把需要的表读进内存：派生规则要跨表看引用关系，不能边读边改。
	rows := map[string][]row{}
	for _, part := range manifest.Parts {
		if part.Kind != "database" {
			continue
		}
		reader, err := archive.OpenPart(part)
		if err != nil {
			log.Fatal(err)
		}
		data, err := io.ReadAll(reader)
		reader.Close()
		if err != nil {
			log.Fatal(err)
		}
		var parsed []row
		if err := json.Unmarshal(data, &parsed); err != nil {
			log.Fatal(err)
		}
		rows[part.Name] = parsed
	}

	dropped := ""
	// 撤销文档对应的分块 ID：索引文档一般只有 chunk_id，靠它才能把索引一起撤掉。
	droppedChunks := map[string]bool{}
	if *mode == "revoke" {
		dropped = smallestID(rows["knowledges"])
		for _, item := range rows["chunks"] {
			if field(item, "knowledge_id") == dropped {
				droppedChunks[field(item, "id")] = true
			}
		}
	}
	// 仍被引用的 handle：被撤销文档的分块与本体都不算引用。
	referenced := map[string]bool{}
	for table, items := range rows {
		if table != "chunks" && table != "knowledges" {
			continue
		}
		for _, item := range items {
			if *mode == "revoke" && (field(item, "id") == dropped || field(item, "knowledge_id") == dropped) {
				continue
			}
			for _, key := range []string{"content", "description", "title"} {
				for _, handle := range extractHandles(field(item, key)) {
					referenced[handle] = true
				}
			}
		}
	}
	// 资源部件名是资源 ID：只有当它的 handle 仍被引用时才保留。
	// 否则撤销后归档会带一个生产作用域算不到的孤儿资源，校验必然失败。
	// 绑定关系同样是"活着"的依据：有些资源只在绑定里出现，正文并不引用。
	bound := map[string]bool{}
	for _, item := range rows["resource_bindings"] {
		if owner := field(item, "owner_id"); owner == dropped {
			continue
		}
		if resource := field(item, "resource_id"); resource != "" {
			bound[resource] = true
		}
	}
	keepResource := map[string]bool{}
	if *mode == "revoke" {
		for _, item := range rows["resources"] {
			id := field(item, "id")
			if referenced[field(item, "handle")] || bound[id] {
				keepResource[id] = true
			}
		}
	}

	parts := make([]librarysnapshot.Part, 0, len(manifest.Parts))
	for _, part := range manifest.Parts {
		reader, err := archive.OpenPart(part)
		if err != nil {
			log.Fatal(err)
		}
		data, err := io.ReadAll(reader)
		reader.Close()
		if err != nil {
			log.Fatal(err)
		}
		switch {
		case *mode == "revoke" && part.Kind == "resource" && !keepResource[part.Name]:
			continue
		case part.Kind == "database" && part.Name == "knowledges":
			data = mutateKnowledges(data, *mode, dropped)
		case *mode == "revoke" && part.Kind == "database" && part.Name == "resources":
			// 孤儿资源的行也要一起删：归档要求"每行资源都有对象部件"，
			// 只删部件会得到一份自相矛盾的快照。
			data = filterResources(data, keepResource)
		case *mode == "revoke" && part.Kind == "database" && part.Name == "chunks":
			data = dropByKnowledge(data, dropped)
		case *mode == "revoke" && part.Kind == "database" && part.Name == "resource_bindings":
			data = dropByOwner(data, dropped)
		case *mode == "revoke" && part.Kind == "index":
			data = dropIndexRows(data, dropped, droppedChunks)
		}
		sealed, err := archive.Put(bytes.NewReader(data))
		if err != nil {
			log.Fatal(err)
		}
		sealed.Kind, sealed.Name, sealed.Count = part.Kind, part.Name, part.Count
		if part.Kind == "database" || part.Kind == "index" {
			count, countErr := rowCount(data)
			if countErr != nil {
				log.Fatal(countErr)
			}
			sealed.Count = count
		}
		parts = append(parts, sealed)
	}
	manifest.Parts = parts
	manifest.CreatedAt = time.Now().UTC()
	id, err := archive.Seal(manifest)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf(`{"snapshot_id":%q,"mode":%q,"base":%q,"dropped":%q,"parts":%d}`, id, *mode, *base, dropped, len(parts))
}

func extractHandles(text string) []string {
	matches := handlePattern.FindAllStringSubmatch(text, -1)
	handles := make([]string, 0, len(matches))
	for _, match := range matches {
		handles = append(handles, match[1])
	}
	return handles
}

func smallestID(items []row) string {
	smallest := ""
	for _, item := range items {
		id := field(item, "id")
		if id != "" && (smallest == "" || id < smallest) {
			smallest = id
		}
	}
	return smallest
}

func field(item row, key string) string {
	raw, ok := item[key]
	if !ok {
		return ""
	}
	var value string
	_ = json.Unmarshal(raw, &value)
	return value
}

// mutateKnowledges：update 改一条标题（内容变更），revoke 删掉指定文档。
func mutateKnowledges(data []byte, mode, dropped string) []byte {
	var items []row
	if err := json.Unmarshal(data, &items); err != nil {
		log.Fatal(err)
	}
	target := smallestID(items)
	for _, item := range items {
		if mode == "update" && field(item, "id") == target {
			title, _ := json.Marshal(field(item, "title") + "（E2E 更新）")
			item["title"] = title
		}
	}
	if mode == "revoke" {
		kept := make([]row, 0, len(items))
		for _, item := range items {
			if field(item, "id") == dropped {
				continue
			}
			kept = append(kept, item)
		}
		return mustJSON(kept)
	}
	return mustJSON(items)
}

// filterResources 只保留仍然被引用的资源行。
func filterResources(data []byte, keep map[string]bool) []byte {
	var items []row
	if err := json.Unmarshal(data, &items); err != nil {
		log.Fatal(err)
	}
	kept := make([]row, 0, len(items))
	for _, item := range items {
		if keep[field(item, "id")] {
			kept = append(kept, item)
		}
	}
	return mustJSON(kept)
}

// dropIndexRows 既按 knowledge_id 也按 chunk_id 删索引文档：
// 真实索引文档通常只带 chunk_id，漏了它就会留下"孤儿向量"。
func dropIndexRows(data []byte, knowledgeID string, chunkIDs map[string]bool) []byte {
	var items []row
	if err := json.Unmarshal(data, &items); err != nil {
		log.Fatal(err)
	}
	kept := make([]row, 0, len(items))
	for _, item := range items {
		// 归档索引行是 {ID, Source} 包装；真正字段在 Source 里。
		source := item
		if raw, ok := item["Source"]; ok {
			var inner row
			if json.Unmarshal(raw, &inner) == nil {
				source = inner
			}
		}
		if field(source, "knowledge_id") == knowledgeID {
			continue
		}
		if chunkIDs[field(source, "chunk_id")] || chunkIDs[field(source, "id")] {
			continue
		}
		kept = append(kept, item)
	}
	return mustJSON(kept)
}

func dropByKnowledge(data []byte, knowledgeID string) []byte {
	var items []row
	if err := json.Unmarshal(data, &items); err != nil {
		log.Fatal(err)
	}
	kept := make([]row, 0, len(items))
	for _, item := range items {
		if field(item, "knowledge_id") == knowledgeID {
			continue
		}
		kept = append(kept, item)
	}
	return mustJSON(kept)
}

func dropByOwner(data []byte, ownerID string) []byte {
	var items []row
	if err := json.Unmarshal(data, &items); err != nil {
		log.Fatal(err)
	}
	kept := make([]row, 0, len(items))
	for _, item := range items {
		if field(item, "owner_id") == ownerID {
			continue
		}
		kept = append(kept, item)
	}
	return mustJSON(kept)
}

func rowCount(data []byte) (int64, error) {
	var items []json.RawMessage
	if err := json.Unmarshal(data, &items); err != nil {
		return 0, err
	}
	return int64(len(items)), nil
}

func mustJSON(value any) []byte {
	data, err := json.Marshal(value)
	if err != nil {
		panic(err)
	}
	return data
}