fe6e218d9c
build / build (push) Successful in 1m17s
候选归档 79 版 / 56MB blob 上,overview、storage、snapshots 三个只读接口 都在 10s 请求期限后返回 503。分段计时定位到唯一瓶颈是按版本重读全部 blob: Open 0ms、LoadManifest×79 11~22ms、ContentUsage 15~25ms、 Usage(逐版 Load 验字节)13.75~14.11s;单版 Load 261~290ms。 - Usage 拆成 Usage(逐版复核字节,同步/恢复与运维 CLI 用)与 UsageFromManifests(同一份去重实现,只做清单级校验),列举路径改用后者; - library_storage.go 每版两次 Load、snapshots 每版一次 Load 改为一次清单读取; - wiki/文档差异只读所需部件,被读部件仍由 OpenPart 复核摘要; - LoadManifest 与 Load 共用同一套清单判据(严格解码 + 重新编码校验); - 列举响应带 verification=manifest / verified=false,详情整版复核后才是 content, 避免把"清单可读"读成"完整性已验证"; - libraryWithDeadline 去掉超时后再写已结束响应的 goroutine 写法 (候选环境实测 http: superfluous response.WriteHeader call)。 回归:多版本共享大 blob 时枚举不读 blob 字节(篡改后枚举仍成功), 而 Load/OpenPart/Usage 仍拒绝坏 blob;坏清单两条路径都整条报错。 make verify 15 项全绿(local 档案)。
422 lines
13 KiB
Go
422 lines
13 KiB
Go
// Package librarysnapshot stores immutable, content-addressed library artifacts.
|
|
// It does not authorize releases or imply cross-system snapshot consistency.
|
|
package librarysnapshot
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"knowledge-governance/internal/releasebundle"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
const Format = "weknora-library-snapshot/v1"
|
|
|
|
// ArtifactAcceptancePolicy requires separate snapshot-bound semantic acceptance.
|
|
const ArtifactAcceptancePolicy = "artifact-acceptance-v1"
|
|
const maxManifestBytes = 16 * 1024 * 1024
|
|
|
|
type Part struct {
|
|
Kind string `json:"kind"`
|
|
Name string `json:"name"`
|
|
SHA256 string `json:"sha256"`
|
|
Bytes int64 `json:"bytes"`
|
|
Count int64 `json:"count"`
|
|
}
|
|
type Compatibility struct {
|
|
WeKnoraVersion string `json:"weknora_version"`
|
|
SchemaVersion int `json:"schema_version"`
|
|
RetrieverEngine string `json:"retriever_engine"`
|
|
EmbeddingModel string `json:"embedding_model"`
|
|
EmbeddingDimension int `json:"embedding_dimension"`
|
|
IndexMappingSHA256 string `json:"index_mapping_sha256"`
|
|
}
|
|
|
|
type Manifest struct {
|
|
ValidationPolicy string `json:"validation_policy,omitempty"`
|
|
Compatibility Compatibility `json:"compatibility"`
|
|
Format string `json:"format"`
|
|
Scope string `json:"scope"`
|
|
TenantID uint64 `json:"tenant_id"`
|
|
KnowledgeBaseID string `json:"knowledge_base_id"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
Parts []Part `json:"parts"`
|
|
}
|
|
type Archive struct {
|
|
root, blobs, versions *os.Root
|
|
maxBlobBytes int64
|
|
}
|
|
|
|
func Open(root string, maxBlobBytes int64) (*Archive, error) {
|
|
if !filepath.IsAbs(root) || maxBlobBytes < 1 || maxBlobBytes == math.MaxInt64 {
|
|
return nil, errors.New("invalid archive configuration")
|
|
}
|
|
info, err := os.Lstat(root)
|
|
if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 {
|
|
return nil, errors.New("archive root must be a private real directory")
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(root)
|
|
if err != nil || resolved != filepath.Clean(root) {
|
|
return nil, errors.New("archive root contains symlinks")
|
|
}
|
|
r, err := os.OpenRoot(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
a := &Archive{root: r, maxBlobBytes: maxBlobBytes}
|
|
for _, dir := range []string{"blobs", "versions"} {
|
|
if err := r.Mkdir(dir, 0700); err != nil && !errors.Is(err, os.ErrExist) {
|
|
a.Close()
|
|
return nil, err
|
|
}
|
|
info, err := r.Lstat(dir)
|
|
if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 {
|
|
a.Close()
|
|
return nil, errors.New("archive subdirectory must be private and real")
|
|
}
|
|
sub, err := r.OpenRoot(dir)
|
|
if err != nil {
|
|
a.Close()
|
|
return nil, err
|
|
}
|
|
if dir == "blobs" {
|
|
a.blobs = sub
|
|
} else {
|
|
a.versions = sub
|
|
}
|
|
}
|
|
return a, nil
|
|
}
|
|
func (a *Archive) Close() error {
|
|
var err error
|
|
for _, r := range []*os.Root{a.versions, a.blobs, a.root} {
|
|
if r != nil {
|
|
err = errors.Join(err, r.Close())
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Put streams bytes into an archive-owned file; it never keeps a source path or
|
|
// signed URL as the recovery dependency. Publishing uses no-overwrite links.
|
|
func (a *Archive) Put(r io.Reader) (Part, error) {
|
|
if r == nil {
|
|
return Part{}, errors.New("nil blob reader")
|
|
}
|
|
name, f, err := temporary(a.blobs)
|
|
if err != nil {
|
|
return Part{}, err
|
|
}
|
|
defer a.blobs.Remove(name)
|
|
defer f.Close()
|
|
h := sha256.New()
|
|
n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(r, a.maxBlobBytes+1))
|
|
if err != nil {
|
|
return Part{}, errors.New("read snapshot blob")
|
|
}
|
|
if n > a.maxBlobBytes {
|
|
return Part{}, errors.New("snapshot blob exceeds configured limit")
|
|
}
|
|
if err = f.Sync(); err != nil {
|
|
return Part{}, err
|
|
}
|
|
if err = f.Close(); err != nil {
|
|
return Part{}, err
|
|
}
|
|
p := Part{SHA256: hex.EncodeToString(h.Sum(nil)), Bytes: n}
|
|
if err = a.blobs.Link(name, p.SHA256); err != nil && !errors.Is(err, os.ErrExist) {
|
|
return Part{}, err
|
|
}
|
|
if err = a.verify(p); err != nil {
|
|
return Part{}, err
|
|
}
|
|
if err = syncRoot(a.blobs); err != nil {
|
|
return Part{}, err
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
// Seal is the commit point: no version is visible until every dependency has
|
|
// been read and hashed locally. Empty tables are explicit, not "leave old data".
|
|
func (a *Archive) Seal(m Manifest) (string, error) {
|
|
data, err := encodeManifest(m)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if err = a.verifyParts(m.Parts); err != nil {
|
|
return "", err
|
|
}
|
|
id := digest(data)
|
|
name, f, err := temporary(a.versions)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer a.versions.Remove(name)
|
|
defer f.Close()
|
|
if _, err = f.Write(data); err != nil {
|
|
return "", err
|
|
}
|
|
if err = f.Sync(); err != nil {
|
|
return "", err
|
|
}
|
|
if err = f.Close(); err != nil {
|
|
return "", err
|
|
}
|
|
if err = a.versions.Link(name, id+".json"); err != nil && !errors.Is(err, os.ErrExist) {
|
|
return "", err
|
|
}
|
|
if _, err = a.Load(id); err != nil {
|
|
return "", err
|
|
}
|
|
if err = syncRoot(a.versions); err != nil {
|
|
return "", err
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// LoadManifest 只校验清单自身(文件名摘要 == 内容摘要 + 结构合法),不重算所有 blob。
|
|
// 用于"盘点/统计/列版本"这类需要遍历全部版本的地方:逐版重算 blob 会把
|
|
// 一次读取放大成 O(全部版本字节),真实踩到 79 版 × 55MB = 13.8s 的接口超时。
|
|
// 真正要用某个版本(同步/恢复)时仍然走 Load,那里会逐部件校验字节。
|
|
//
|
|
// 它证明的只有"清单可读且结构合法",**不证明**这份版本引用的字节当前完好:
|
|
// 调用方不得把 LoadManifest 的结果标成"完整性已验证"或"可发布"。
|
|
func (a *Archive) LoadManifest(id string) (Manifest, error) {
|
|
if !validDigest(id) {
|
|
return Manifest{}, errors.New("invalid snapshot ID")
|
|
}
|
|
f, err := regular(a.versions, id+".json", maxManifestBytes)
|
|
if err != nil {
|
|
return Manifest{}, err
|
|
}
|
|
defer f.Close()
|
|
data, err := io.ReadAll(io.LimitReader(f, maxManifestBytes+1))
|
|
if err != nil || int64(len(data)) > maxManifestBytes || digest(data) != id {
|
|
return Manifest{}, errors.New("snapshot manifest digest mismatch")
|
|
}
|
|
dec := json.NewDecoder(bytes.NewReader(data))
|
|
dec.DisallowUnknownFields()
|
|
var m Manifest
|
|
if err = dec.Decode(&m); err != nil {
|
|
return Manifest{}, errors.New("invalid snapshot manifest")
|
|
}
|
|
var extra any
|
|
if dec.Decode(&extra) != io.EOF {
|
|
return Manifest{}, errors.New("trailing snapshot manifest data")
|
|
}
|
|
if _, err = encodeManifest(m); err != nil {
|
|
return Manifest{}, err
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
// Load revalidates bytes, not just the manifest's claim of recoverability.
|
|
func (a *Archive) Load(id string) (Manifest, error) {
|
|
m, err := a.LoadManifest(id)
|
|
if err != nil {
|
|
return Manifest{}, err
|
|
}
|
|
if err = a.verifyParts(m.Parts); err != nil {
|
|
return Manifest{}, err
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
func (a *Archive) verifyParts(parts []Part) error {
|
|
seen := map[string]int64{}
|
|
counts := map[string]int64{}
|
|
resourceIDs := map[string]bool{}
|
|
archivedResources := map[string]bool{}
|
|
for _, p := range parts {
|
|
if size, ok := seen[p.SHA256]; ok {
|
|
if size != p.Bytes {
|
|
return errors.New("conflicting blob sizes")
|
|
}
|
|
} else {
|
|
if err := a.verify(p); err != nil {
|
|
return err
|
|
}
|
|
seen[p.SHA256] = p.Bytes
|
|
}
|
|
if p.Kind == "resource" {
|
|
archivedResources[p.Name] = true
|
|
continue
|
|
}
|
|
count, known := counts[p.SHA256]
|
|
if !known || p.Kind == "database" && p.Name == "resources" {
|
|
var err error
|
|
count, err = a.countRows(p, resourceIDs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
counts[p.SHA256] = count
|
|
}
|
|
if count != p.Count {
|
|
return errors.New("snapshot count does not match archived rows")
|
|
}
|
|
}
|
|
if len(resourceIDs) != len(archivedResources) {
|
|
return errors.New("resource archive dependencies incomplete")
|
|
}
|
|
for id := range resourceIDs {
|
|
if !archivedResources[id] {
|
|
return errors.New("resource bytes not archived")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (a *Archive) countRows(p Part, resources map[string]bool) (int64, error) {
|
|
f, err := regular(a.blobs, p.SHA256, a.maxBlobBytes)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer f.Close()
|
|
dec := json.NewDecoder(io.LimitReader(f, a.maxBlobBytes+1))
|
|
token, err := dec.Token()
|
|
if err != nil || token != json.Delim('[') {
|
|
return 0, errors.New("snapshot component must be a JSON row array")
|
|
}
|
|
var n int64
|
|
for dec.More() {
|
|
var row map[string]json.RawMessage
|
|
if err = dec.Decode(&row); err != nil || row == nil {
|
|
return 0, errors.New("invalid archived row")
|
|
}
|
|
n++
|
|
if p.Kind == "database" && p.Name == "resources" {
|
|
var id string
|
|
if json.Unmarshal(row["id"], &id) != nil || !identifier(id) || resources[id] {
|
|
return 0, errors.New("invalid resource archive identity")
|
|
}
|
|
resources[id] = true
|
|
}
|
|
}
|
|
if _, err = dec.Token(); err != nil {
|
|
return 0, errors.New("incomplete archived row array")
|
|
}
|
|
var extra any
|
|
if dec.Decode(&extra) != io.EOF {
|
|
return 0, errors.New("trailing archived row data")
|
|
}
|
|
return n, nil
|
|
}
|
|
func (a *Archive) verify(p Part) error {
|
|
if !validDigest(p.SHA256) || p.Bytes < 0 || p.Bytes > a.maxBlobBytes {
|
|
return errors.New("invalid snapshot blob descriptor")
|
|
}
|
|
f, err := regular(a.blobs, p.SHA256, a.maxBlobBytes)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
h := sha256.New()
|
|
n, err := io.Copy(h, io.LimitReader(f, a.maxBlobBytes+1))
|
|
if err != nil || n != p.Bytes || hex.EncodeToString(h.Sum(nil)) != p.SHA256 {
|
|
return errors.New("snapshot blob integrity mismatch")
|
|
}
|
|
return nil
|
|
}
|
|
func regular(root *os.Root, name string, limit int64) (*os.File, error) {
|
|
f, err := root.OpenFile(name, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
if err != nil {
|
|
return nil, errors.New("snapshot file unavailable")
|
|
}
|
|
st, err := f.Stat()
|
|
if err != nil || !st.Mode().IsRegular() || st.Mode().Perm() != 0600 || st.Size() > limit {
|
|
f.Close()
|
|
return nil, errors.New("unsafe snapshot file")
|
|
}
|
|
return f, nil
|
|
}
|
|
func temporary(root *os.Root) (string, *os.File, error) {
|
|
var random [16]byte
|
|
if _, err := rand.Read(random[:]); err != nil {
|
|
return "", nil, err
|
|
}
|
|
name := ".pending-" + hex.EncodeToString(random[:])
|
|
f, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
|
return name, f, err
|
|
}
|
|
func syncRoot(root *os.Root) error {
|
|
f, err := root.Open(".")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
return f.Sync()
|
|
}
|
|
func digest(data []byte) string { h := sha256.Sum256(data); return hex.EncodeToString(h[:]) }
|
|
func validDigest(s string) bool {
|
|
if len(s) != 64 {
|
|
return false
|
|
}
|
|
for _, c := range s {
|
|
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
func identifier(s string) bool {
|
|
if len(s) == 0 || len(s) > 128 {
|
|
return false
|
|
}
|
|
for _, c := range s {
|
|
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '-' || c == '_' || c == '.') {
|
|
return false
|
|
}
|
|
}
|
|
return s != "." && s != ".."
|
|
}
|
|
|
|
func encodeManifest(m Manifest) ([]byte, error) {
|
|
if m.ValidationPolicy != "" && m.ValidationPolicy != ArtifactAcceptancePolicy {
|
|
return nil, errors.New("unsupported snapshot validation policy")
|
|
}
|
|
c := m.Compatibility
|
|
if c.WeKnoraVersion != releasebundle.WeKnoraVersionV071 || c.SchemaVersion != releasebundle.WeKnoraMigrationV071 || c.RetrieverEngine != "elasticsearch_v8" || !identifier(c.EmbeddingModel) || c.EmbeddingDimension != releasebundle.EmbeddingDimensionV071 || !validDigest(c.IndexMappingSHA256) {
|
|
return nil, errors.New("unsupported snapshot compatibility contract")
|
|
}
|
|
if m.Format != Format || (m.Scope != "internal" && m.Scope != "external") || m.TenantID == 0 || !identifier(m.KnowledgeBaseID) || m.CreatedAt.IsZero() {
|
|
return nil, errors.New("invalid library manifest identity")
|
|
}
|
|
required := map[string]bool{}
|
|
for _, n := range []string{"knowledges", "chunks", "resources", "resource_bindings", "knowledge_tags", "knowledge_tag_relations"} {
|
|
required["database/"+n] = true
|
|
}
|
|
for _, n := range []string{"wiki_pages", "wiki_folders", "wiki_page_issues", "wiki_log_entries"} {
|
|
required["wiki/"+n] = true
|
|
}
|
|
required["index/chunks"] = true
|
|
seen := map[string]bool{}
|
|
for _, p := range m.Parts {
|
|
key := p.Kind + "/" + p.Name
|
|
if (!required[key] && !(p.Kind == "resource" && identifier(p.Name) && p.Count == 1)) || seen[key] || !validDigest(p.SHA256) || p.Bytes < 0 || p.Count < 0 {
|
|
return nil, errors.New("invalid or nonknowledge snapshot component")
|
|
}
|
|
seen[key] = true
|
|
}
|
|
for key := range required {
|
|
if !seen[key] {
|
|
return nil, errors.New("incomplete library snapshot")
|
|
}
|
|
}
|
|
m.Parts = append([]Part(nil), m.Parts...)
|
|
sort.Slice(m.Parts, func(i, j int) bool { return m.Parts[i].Kind+"/"+m.Parts[i].Name < m.Parts[j].Kind+"/"+m.Parts[j].Name })
|
|
data, err := json.Marshal(m)
|
|
if err != nil || len(data) > maxManifestBytes {
|
|
return nil, errors.New("snapshot manifest too large")
|
|
}
|
|
return data, nil
|
|
}
|