Files
knowledge-governance/internal/librarysnapshot/archive.go
T
姚章浩 fe6e218d9c
build / build (push) Successful in 1m17s
fix(library): 整库只读接口改清单级读取,修掉真实超时
候选归档 79 版 / 56MB blob 上,overview、storage、snapshots 三个只读接口
都在 10s 请求期限后返回 503。分段计时定位到唯一瓶颈是按版本重读全部 blob:
Open 0ms、LoadManifest×79 11~22ms、ContentUsage 15~25ms、
Usage(逐版 Load 验字节)13.75~14.11s;单版 Load 261~290ms。

- Usage 拆成 Usage(逐版复核字节,同步/恢复与运维 CLI 用)与
  UsageFromManifests(同一份去重实现,只做清单级校验),列举路径改用后者;
- library_storage.go 每版两次 Load、snapshots 每版一次 Load 改为一次清单读取;
- wiki/文档差异只读所需部件,被读部件仍由 OpenPart 复核摘要;
- LoadManifest 与 Load 共用同一套清单判据(严格解码 + 重新编码校验);
- 列举响应带 verification=manifest / verified=false,详情整版复核后才是 content,
  避免把"清单可读"读成"完整性已验证";
- libraryWithDeadline 去掉超时后再写已结束响应的 goroutine 写法
  (候选环境实测 http: superfluous response.WriteHeader call)。

回归:多版本共享大 blob 时枚举不读 blob 字节(篡改后枚举仍成功),
而 Load/OpenPart/Usage 仍拒绝坏 blob;坏清单两条路径都整条报错。
make verify 15 项全绿(local 档案)。
2026-10-28 21:52:14 +08:00

422 lines
13 KiB
Go

// Package librarysnapshot stores immutable, content-addressed library artifacts.
// It does not authorize releases or imply cross-system snapshot consistency.
package librarysnapshot
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"knowledge-governance/internal/releasebundle"
"math"
"os"
"path/filepath"
"sort"
"syscall"
"time"
)
const Format = "weknora-library-snapshot/v1"
// ArtifactAcceptancePolicy requires separate snapshot-bound semantic acceptance.
const ArtifactAcceptancePolicy = "artifact-acceptance-v1"
const maxManifestBytes = 16 * 1024 * 1024
type Part struct {
Kind string `json:"kind"`
Name string `json:"name"`
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
Count int64 `json:"count"`
}
type Compatibility struct {
WeKnoraVersion string `json:"weknora_version"`
SchemaVersion int `json:"schema_version"`
RetrieverEngine string `json:"retriever_engine"`
EmbeddingModel string `json:"embedding_model"`
EmbeddingDimension int `json:"embedding_dimension"`
IndexMappingSHA256 string `json:"index_mapping_sha256"`
}
type Manifest struct {
ValidationPolicy string `json:"validation_policy,omitempty"`
Compatibility Compatibility `json:"compatibility"`
Format string `json:"format"`
Scope string `json:"scope"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
CreatedAt time.Time `json:"created_at"`
Parts []Part `json:"parts"`
}
type Archive struct {
root, blobs, versions *os.Root
maxBlobBytes int64
}
func Open(root string, maxBlobBytes int64) (*Archive, error) {
if !filepath.IsAbs(root) || maxBlobBytes < 1 || maxBlobBytes == math.MaxInt64 {
return nil, errors.New("invalid archive configuration")
}
info, err := os.Lstat(root)
if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 {
return nil, errors.New("archive root must be a private real directory")
}
resolved, err := filepath.EvalSymlinks(root)
if err != nil || resolved != filepath.Clean(root) {
return nil, errors.New("archive root contains symlinks")
}
r, err := os.OpenRoot(root)
if err != nil {
return nil, err
}
a := &Archive{root: r, maxBlobBytes: maxBlobBytes}
for _, dir := range []string{"blobs", "versions"} {
if err := r.Mkdir(dir, 0700); err != nil && !errors.Is(err, os.ErrExist) {
a.Close()
return nil, err
}
info, err := r.Lstat(dir)
if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 {
a.Close()
return nil, errors.New("archive subdirectory must be private and real")
}
sub, err := r.OpenRoot(dir)
if err != nil {
a.Close()
return nil, err
}
if dir == "blobs" {
a.blobs = sub
} else {
a.versions = sub
}
}
return a, nil
}
func (a *Archive) Close() error {
var err error
for _, r := range []*os.Root{a.versions, a.blobs, a.root} {
if r != nil {
err = errors.Join(err, r.Close())
}
}
return err
}
// Put streams bytes into an archive-owned file; it never keeps a source path or
// signed URL as the recovery dependency. Publishing uses no-overwrite links.
func (a *Archive) Put(r io.Reader) (Part, error) {
if r == nil {
return Part{}, errors.New("nil blob reader")
}
name, f, err := temporary(a.blobs)
if err != nil {
return Part{}, err
}
defer a.blobs.Remove(name)
defer f.Close()
h := sha256.New()
n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(r, a.maxBlobBytes+1))
if err != nil {
return Part{}, errors.New("read snapshot blob")
}
if n > a.maxBlobBytes {
return Part{}, errors.New("snapshot blob exceeds configured limit")
}
if err = f.Sync(); err != nil {
return Part{}, err
}
if err = f.Close(); err != nil {
return Part{}, err
}
p := Part{SHA256: hex.EncodeToString(h.Sum(nil)), Bytes: n}
if err = a.blobs.Link(name, p.SHA256); err != nil && !errors.Is(err, os.ErrExist) {
return Part{}, err
}
if err = a.verify(p); err != nil {
return Part{}, err
}
if err = syncRoot(a.blobs); err != nil {
return Part{}, err
}
return p, nil
}
// Seal is the commit point: no version is visible until every dependency has
// been read and hashed locally. Empty tables are explicit, not "leave old data".
func (a *Archive) Seal(m Manifest) (string, error) {
data, err := encodeManifest(m)
if err != nil {
return "", err
}
if err = a.verifyParts(m.Parts); err != nil {
return "", err
}
id := digest(data)
name, f, err := temporary(a.versions)
if err != nil {
return "", err
}
defer a.versions.Remove(name)
defer f.Close()
if _, err = f.Write(data); err != nil {
return "", err
}
if err = f.Sync(); err != nil {
return "", err
}
if err = f.Close(); err != nil {
return "", err
}
if err = a.versions.Link(name, id+".json"); err != nil && !errors.Is(err, os.ErrExist) {
return "", err
}
if _, err = a.Load(id); err != nil {
return "", err
}
if err = syncRoot(a.versions); err != nil {
return "", err
}
return id, nil
}
// LoadManifest 只校验清单自身(文件名摘要 == 内容摘要 + 结构合法),不重算所有 blob。
// 用于"盘点/统计/列版本"这类需要遍历全部版本的地方:逐版重算 blob 会把
// 一次读取放大成 O(全部版本字节),真实踩到 79 版 × 55MB = 13.8s 的接口超时。
// 真正要用某个版本(同步/恢复)时仍然走 Load,那里会逐部件校验字节。
//
// 它证明的只有"清单可读且结构合法",**不证明**这份版本引用的字节当前完好:
// 调用方不得把 LoadManifest 的结果标成"完整性已验证"或"可发布"。
func (a *Archive) LoadManifest(id string) (Manifest, error) {
if !validDigest(id) {
return Manifest{}, errors.New("invalid snapshot ID")
}
f, err := regular(a.versions, id+".json", maxManifestBytes)
if err != nil {
return Manifest{}, err
}
defer f.Close()
data, err := io.ReadAll(io.LimitReader(f, maxManifestBytes+1))
if err != nil || int64(len(data)) > maxManifestBytes || digest(data) != id {
return Manifest{}, errors.New("snapshot manifest digest mismatch")
}
dec := json.NewDecoder(bytes.NewReader(data))
dec.DisallowUnknownFields()
var m Manifest
if err = dec.Decode(&m); err != nil {
return Manifest{}, errors.New("invalid snapshot manifest")
}
var extra any
if dec.Decode(&extra) != io.EOF {
return Manifest{}, errors.New("trailing snapshot manifest data")
}
if _, err = encodeManifest(m); err != nil {
return Manifest{}, err
}
return m, nil
}
// Load revalidates bytes, not just the manifest's claim of recoverability.
func (a *Archive) Load(id string) (Manifest, error) {
m, err := a.LoadManifest(id)
if err != nil {
return Manifest{}, err
}
if err = a.verifyParts(m.Parts); err != nil {
return Manifest{}, err
}
return m, nil
}
func (a *Archive) verifyParts(parts []Part) error {
seen := map[string]int64{}
counts := map[string]int64{}
resourceIDs := map[string]bool{}
archivedResources := map[string]bool{}
for _, p := range parts {
if size, ok := seen[p.SHA256]; ok {
if size != p.Bytes {
return errors.New("conflicting blob sizes")
}
} else {
if err := a.verify(p); err != nil {
return err
}
seen[p.SHA256] = p.Bytes
}
if p.Kind == "resource" {
archivedResources[p.Name] = true
continue
}
count, known := counts[p.SHA256]
if !known || p.Kind == "database" && p.Name == "resources" {
var err error
count, err = a.countRows(p, resourceIDs)
if err != nil {
return err
}
counts[p.SHA256] = count
}
if count != p.Count {
return errors.New("snapshot count does not match archived rows")
}
}
if len(resourceIDs) != len(archivedResources) {
return errors.New("resource archive dependencies incomplete")
}
for id := range resourceIDs {
if !archivedResources[id] {
return errors.New("resource bytes not archived")
}
}
return nil
}
func (a *Archive) countRows(p Part, resources map[string]bool) (int64, error) {
f, err := regular(a.blobs, p.SHA256, a.maxBlobBytes)
if err != nil {
return 0, err
}
defer f.Close()
dec := json.NewDecoder(io.LimitReader(f, a.maxBlobBytes+1))
token, err := dec.Token()
if err != nil || token != json.Delim('[') {
return 0, errors.New("snapshot component must be a JSON row array")
}
var n int64
for dec.More() {
var row map[string]json.RawMessage
if err = dec.Decode(&row); err != nil || row == nil {
return 0, errors.New("invalid archived row")
}
n++
if p.Kind == "database" && p.Name == "resources" {
var id string
if json.Unmarshal(row["id"], &id) != nil || !identifier(id) || resources[id] {
return 0, errors.New("invalid resource archive identity")
}
resources[id] = true
}
}
if _, err = dec.Token(); err != nil {
return 0, errors.New("incomplete archived row array")
}
var extra any
if dec.Decode(&extra) != io.EOF {
return 0, errors.New("trailing archived row data")
}
return n, nil
}
func (a *Archive) verify(p Part) error {
if !validDigest(p.SHA256) || p.Bytes < 0 || p.Bytes > a.maxBlobBytes {
return errors.New("invalid snapshot blob descriptor")
}
f, err := regular(a.blobs, p.SHA256, a.maxBlobBytes)
if err != nil {
return err
}
defer f.Close()
h := sha256.New()
n, err := io.Copy(h, io.LimitReader(f, a.maxBlobBytes+1))
if err != nil || n != p.Bytes || hex.EncodeToString(h.Sum(nil)) != p.SHA256 {
return errors.New("snapshot blob integrity mismatch")
}
return nil
}
func regular(root *os.Root, name string, limit int64) (*os.File, error) {
f, err := root.OpenFile(name, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, errors.New("snapshot file unavailable")
}
st, err := f.Stat()
if err != nil || !st.Mode().IsRegular() || st.Mode().Perm() != 0600 || st.Size() > limit {
f.Close()
return nil, errors.New("unsafe snapshot file")
}
return f, nil
}
func temporary(root *os.Root) (string, *os.File, error) {
var random [16]byte
if _, err := rand.Read(random[:]); err != nil {
return "", nil, err
}
name := ".pending-" + hex.EncodeToString(random[:])
f, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
return name, f, err
}
func syncRoot(root *os.Root) error {
f, err := root.Open(".")
if err != nil {
return err
}
defer f.Close()
return f.Sync()
}
func digest(data []byte) string { h := sha256.Sum256(data); return hex.EncodeToString(h[:]) }
func validDigest(s string) bool {
if len(s) != 64 {
return false
}
for _, c := range s {
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
return false
}
}
return true
}
func identifier(s string) bool {
if len(s) == 0 || len(s) > 128 {
return false
}
for _, c := range s {
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '-' || c == '_' || c == '.') {
return false
}
}
return s != "." && s != ".."
}
func encodeManifest(m Manifest) ([]byte, error) {
if m.ValidationPolicy != "" && m.ValidationPolicy != ArtifactAcceptancePolicy {
return nil, errors.New("unsupported snapshot validation policy")
}
c := m.Compatibility
if c.WeKnoraVersion != releasebundle.WeKnoraVersionV071 || c.SchemaVersion != releasebundle.WeKnoraMigrationV071 || c.RetrieverEngine != "elasticsearch_v8" || !identifier(c.EmbeddingModel) || c.EmbeddingDimension != releasebundle.EmbeddingDimensionV071 || !validDigest(c.IndexMappingSHA256) {
return nil, errors.New("unsupported snapshot compatibility contract")
}
if m.Format != Format || (m.Scope != "internal" && m.Scope != "external") || m.TenantID == 0 || !identifier(m.KnowledgeBaseID) || m.CreatedAt.IsZero() {
return nil, errors.New("invalid library manifest identity")
}
required := map[string]bool{}
for _, n := range []string{"knowledges", "chunks", "resources", "resource_bindings", "knowledge_tags", "knowledge_tag_relations"} {
required["database/"+n] = true
}
for _, n := range []string{"wiki_pages", "wiki_folders", "wiki_page_issues", "wiki_log_entries"} {
required["wiki/"+n] = true
}
required["index/chunks"] = true
seen := map[string]bool{}
for _, p := range m.Parts {
key := p.Kind + "/" + p.Name
if (!required[key] && !(p.Kind == "resource" && identifier(p.Name) && p.Count == 1)) || seen[key] || !validDigest(p.SHA256) || p.Bytes < 0 || p.Count < 0 {
return nil, errors.New("invalid or nonknowledge snapshot component")
}
seen[key] = true
}
for key := range required {
if !seen[key] {
return nil, errors.New("incomplete library snapshot")
}
}
m.Parts = append([]Part(nil), m.Parts...)
sort.Slice(m.Parts, func(i, j int) bool { return m.Parts[i].Kind+"/"+m.Parts[i].Name < m.Parts[j].Kind+"/"+m.Parts[j].Name })
data, err := json.Marshal(m)
if err != nil || len(data) > maxManifestBytes {
return nil, errors.New("snapshot manifest too large")
}
return data, nil
}