9300865d23
父独立审查退回的四个 finding,逐项修在本包内,不改架构。 1 复用路径没复验字节:remoteEvidence 只读清单+摘要就宣布可用。现在新增 libraryctl snapshot-verify(整版逐部件重算)并在复用前调用;blob 缺失/篡改必须 在入发布队列之前失败。新增真实 local 跨进程用例:清单不动、删 blob / 改 blob 后 再 Push 必须拒绝。 2 回传枚举被固定 LIMIT 50 截断:LibraryJobSnapshots 只取成功作业且按摘要字典序。 改为 LibrarySnapshotCandidates:合并 library_versions、library_jobs.snapshot_id、 library_jobs.target_snapshot_id 三处引用,按 (created_at, snapshot_id) 游标分页; 回传循环翻页到不足一页为止。失败/待恢复作业留下的预备份、第 51 条以后、历史恢复 输入快照都能被枚举。另修"本地只有清单、blob 已损坏就永远跳过":改用整版 Load 判定, 并给 Unpack 增加显式 RepairCorruptBlobs(只在权威云端读回时开启)。 3 身份映射错误:搬运拿目标 tenant/KB 去核对封存快照,而封存清单写的是**源**身份 (云测试验收目标 a8e9c8ad/2e980686 与源 KB 必然不同)。现在 expectedSnapshotIdentity 按版本台账判定:test_sealed → 用部署注入的源绑定(LIBRARY_SOURCE_TENANT_ID / LIBRARY_SOURCE_KB_*);预备份/生效/恢复版本 → 用目标绑定;没有登记或两类冲突一律拒绝。 身份永远来自独立绑定与台账,不接受包自证。 4 两步入队有竞态:Enqueue 成功后云执行器可能立刻领取并开始维护,Attach 失败却把作业 标失败并声称"未维护"。改为 EnqueueAndAttachLibraryJob 单事务(锁行 + 插入 + 关联 + 状态守卫),要么都成功要么都不发生;重复执行幂等,已关联别的队列作业则拒绝。 另修终态重放:AdvanceLibraryJob 无状态守卫,迟到的失败出口会把已成功作业改写; 新增 AdvanceLibraryJobIfActive(仅 pending/running 可推进),拿不到就明确记录"未覆盖"。 测试:libraryhandoff 用例覆盖跨租户/错误 scope/清单在而字节坏/本地损坏自愈; 隔离 fixture 库覆盖"校验通过才入队、失败不入队"、>50 条分页与失败作业预备份、 幂等重放不重复投递、迟到失败不改写终态。 未部署、未推送、未构建镜像;真实跨主机 SSH 链路仍未验证。
388 lines
14 KiB
Go
388 lines
14 KiB
Go
package librarysnapshot
|
||
|
||
import (
|
||
"archive/tar"
|
||
"bytes"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"syscall"
|
||
)
|
||
|
||
// 跨主机搬运一份已封存快照。
|
||
//
|
||
// 为什么不是"把 versions/ 和 blobs/ 目录 rsync 过去":
|
||
// - 直接同步目录时,"版本清单先到、blob 还在路上"的瞬间看起来就是一份可用版本,
|
||
// 下一个发布作业会读到缺字节的档然后把失败记在目标身上;
|
||
// - 也不能只搬 blobs:清单之外的 blob 无法证明属于这份内容。
|
||
//
|
||
// 所以搬运单位是"自描述包":清单先写、每个 blob 以 sha256 命名、下载侧边收边算,
|
||
// 只有全部齐了才把版本清单落位。版本清单文件就是提交点(见 Install 的注释)。
|
||
const BundleFormat = "weknora-library-snapshot-bundle/v1"
|
||
|
||
// 包内路径。清单带身份(文件名即快照 ID),blob 带内容身份(文件名即 sha256):
|
||
// 接收侧不需要额外的锁文件或元数据就能判断"这是什么、字节对不对"。
|
||
const (
|
||
bundleManifestPrefix = "manifest/"
|
||
bundleBlobPrefix = "blobs/"
|
||
maxBundleEntries = 200000
|
||
)
|
||
|
||
// BundleInfo 是一次导出的结果(CLI 打印它,父据此核对哈希)。
|
||
type BundleInfo struct {
|
||
Format string `json:"format"`
|
||
Snapshot string `json:"snapshot"`
|
||
Parts int `json:"parts"`
|
||
Bytes int64 `json:"bytes"`
|
||
// Blobs 是逐个 blob 的 sha256 列表,便于在两台机器上对拍而不传大文件。
|
||
Blobs []string `json:"blobs"`
|
||
}
|
||
|
||
// Pack 把一份已封存版本导出成单文件包(tar)。
|
||
//
|
||
// 导出前先 Load:它逐 blob 重算 sha256,所以"包里的每个字节都在这台机器上验证过"。
|
||
// 导出后不删源,也不改归档。
|
||
func (a *Archive) Pack(id, outPath string) (BundleInfo, error) {
|
||
if !validDigest(id) {
|
||
return BundleInfo{}, errors.New("invalid snapshot ID")
|
||
}
|
||
if !filepath.IsAbs(outPath) || filepath.Clean(outPath) != outPath {
|
||
return BundleInfo{}, errors.New("bundle path must be an absolute clean path")
|
||
}
|
||
manifest, err := a.Load(id)
|
||
if err != nil {
|
||
return BundleInfo{}, fmt.Errorf("snapshot is not verifiable on this host: %w", err)
|
||
}
|
||
manifestFile, err := regular(a.versions, id+".json", maxManifestBytes)
|
||
if err != nil {
|
||
return BundleInfo{}, err
|
||
}
|
||
manifestBytes, readErr := io.ReadAll(io.LimitReader(manifestFile, maxManifestBytes+1))
|
||
manifestFile.Close()
|
||
if readErr != nil || int64(len(manifestBytes)) > maxManifestBytes || digest(manifestBytes) != id {
|
||
return BundleInfo{}, errors.New("snapshot manifest changed while packing")
|
||
}
|
||
if _, err := os.Lstat(outPath); err == nil {
|
||
return BundleInfo{}, errors.New("bundle already exists; refusing to overwrite")
|
||
} else if !errors.Is(err, os.ErrNotExist) {
|
||
return BundleInfo{}, errors.New("bundle path is not usable")
|
||
}
|
||
out, err := os.OpenFile(outPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||
if err != nil {
|
||
return BundleInfo{}, errors.New("create bundle")
|
||
}
|
||
cleanup := func(cause error) (BundleInfo, error) {
|
||
out.Close()
|
||
os.Remove(outPath)
|
||
return BundleInfo{}, cause
|
||
}
|
||
writer := tar.NewWriter(out)
|
||
writeEntry := func(name string, size int64, body io.Reader) error {
|
||
header := &tar.Header{Name: name, Mode: 0600, Size: size, Typeflag: tar.TypeReg}
|
||
if err := writer.WriteHeader(header); err != nil {
|
||
return err
|
||
}
|
||
_, err := io.CopyN(writer, body, size)
|
||
return err
|
||
}
|
||
// 清单必须第一个写:接收侧读到清单才知道后面应该有哪些 blob。
|
||
if err := writeEntry(bundleManifestPrefix+id+".json", int64(len(manifestBytes)), bytes.NewReader(manifestBytes)); err != nil {
|
||
return cleanup(errors.New("write bundle manifest"))
|
||
}
|
||
info := BundleInfo{Format: BundleFormat, Snapshot: id, Parts: len(manifest.Parts)}
|
||
blobs := make([]string, 0, len(manifest.Parts))
|
||
seen := map[string]bool{}
|
||
for _, part := range manifest.Parts {
|
||
if seen[part.SHA256] {
|
||
continue
|
||
}
|
||
seen[part.SHA256] = true
|
||
file, err := regular(a.blobs, part.SHA256, a.maxBlobBytes)
|
||
if err != nil {
|
||
return cleanup(fmt.Errorf("snapshot blob %s is not readable: %w", part.SHA256, err))
|
||
}
|
||
writeErr := writeEntry(bundleBlobPrefix+part.SHA256, part.Bytes, file)
|
||
file.Close()
|
||
if writeErr != nil {
|
||
return cleanup(errors.New("write bundle blob"))
|
||
}
|
||
info.Bytes += part.Bytes
|
||
blobs = append(blobs, part.SHA256)
|
||
}
|
||
if err := writer.Close(); err != nil {
|
||
return cleanup(errors.New("finalize bundle"))
|
||
}
|
||
if err := out.Sync(); err != nil {
|
||
return cleanup(errors.New("sync bundle"))
|
||
}
|
||
if err := out.Close(); err != nil {
|
||
return BundleInfo{}, errors.New("close bundle")
|
||
}
|
||
sort.Strings(blobs)
|
||
info.Blobs = blobs
|
||
return info, nil
|
||
}
|
||
|
||
// Unpack 校验一个包并把它装入本归档。返回值是这份快照自己的清单。
|
||
//
|
||
// 提交点是"版本清单落位":blob 先按内容寻址写好并 fsync,最后一步才 Link
|
||
// versions/<id>.json。因此任何中断都只会留下不可见的 blob(可被下一次重放复用),
|
||
// 不会留下"看起来存在但缺字节"的版本。
|
||
//
|
||
// 幂等:同一份包重复导入成功;名字相同内容不同的 blob 或清单直接拒绝。
|
||
// UnpackOptions 控制"本地已有同名 blob 但字节不符"时的行为。
|
||
//
|
||
// 默认拒绝(同 ID 异内容一律当身份冲突处理)。只有"从权威云端副本读回"这一条路径
|
||
// 显式打开修复:那时同名 blob 字节不符只能是本地损坏,拒绝会让这份版本永远不可读。
|
||
type UnpackOptions struct {
|
||
RepairCorruptBlobs bool
|
||
}
|
||
|
||
func (a *Archive) Unpack(bundlePath, expectID string) (Manifest, error) {
|
||
return a.UnpackWithOptions(bundlePath, expectID, UnpackOptions{})
|
||
}
|
||
|
||
func (a *Archive) UnpackWithOptions(bundlePath, expectID string, options UnpackOptions) (Manifest, error) {
|
||
if expectID != "" && !validDigest(expectID) {
|
||
return Manifest{}, errors.New("invalid expected snapshot ID")
|
||
}
|
||
if !filepath.IsAbs(bundlePath) || filepath.Clean(bundlePath) != bundlePath {
|
||
return Manifest{}, errors.New("bundle path must be an absolute clean path")
|
||
}
|
||
file, err := os.OpenFile(bundlePath, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
||
if err != nil {
|
||
return Manifest{}, errors.New("open bundle")
|
||
}
|
||
defer file.Close()
|
||
if info, statErr := file.Stat(); statErr != nil || !info.Mode().IsRegular() {
|
||
return Manifest{}, errors.New("bundle must be a regular file")
|
||
}
|
||
reader := tar.NewReader(file)
|
||
var (
|
||
id string
|
||
manifest Manifest
|
||
manifestRaw []byte
|
||
expected map[string]Part
|
||
received map[string]bool
|
||
entryCount int
|
||
)
|
||
for {
|
||
header, err := reader.Next()
|
||
if errors.Is(err, io.EOF) {
|
||
break
|
||
}
|
||
if err != nil {
|
||
return Manifest{}, errors.New("bundle is not a readable archive")
|
||
}
|
||
entryCount++
|
||
if entryCount > maxBundleEntries {
|
||
return Manifest{}, errors.New("bundle has too many entries")
|
||
}
|
||
if header.Typeflag != tar.TypeReg {
|
||
return Manifest{}, fmt.Errorf("bundle entry %q is not a regular file", header.Name)
|
||
}
|
||
switch {
|
||
case len(header.Name) == len(bundleManifestPrefix)+len(".json")+64 && header.Name[:len(bundleManifestPrefix)] == bundleManifestPrefix:
|
||
if id != "" {
|
||
return Manifest{}, errors.New("bundle contains more than one manifest")
|
||
}
|
||
id = header.Name[len(bundleManifestPrefix) : len(header.Name)-len(".json")]
|
||
if !validDigest(id) {
|
||
return Manifest{}, errors.New("bundle manifest name is not a snapshot ID")
|
||
}
|
||
if expectID != "" && id != expectID {
|
||
return Manifest{}, fmt.Errorf("bundle snapshot is %s, expected %s", id, expectID)
|
||
}
|
||
if header.Size <= 0 || header.Size > maxManifestBytes {
|
||
return Manifest{}, errors.New("bundle manifest size is not allowed")
|
||
}
|
||
manifestRaw, err = io.ReadAll(io.LimitReader(reader, maxManifestBytes+1))
|
||
if err != nil || int64(len(manifestRaw)) != header.Size || digest(manifestRaw) != id {
|
||
return Manifest{}, errors.New("bundle manifest digest does not match its ID")
|
||
}
|
||
manifest, expected, err = decodeBundleManifest(manifestRaw, id)
|
||
if err != nil {
|
||
return Manifest{}, err
|
||
}
|
||
received = map[string]bool{}
|
||
case len(header.Name) == len(bundleBlobPrefix)+64 && header.Name[:len(bundleBlobPrefix)] == bundleBlobPrefix:
|
||
if id == "" {
|
||
return Manifest{}, errors.New("bundle blob appears before the manifest")
|
||
}
|
||
name := header.Name[len(bundleBlobPrefix):]
|
||
if !validDigest(name) {
|
||
return Manifest{}, errors.New("bundle blob name is not a sha256 digest")
|
||
}
|
||
part, ok := expected[name]
|
||
if !ok {
|
||
return Manifest{}, fmt.Errorf("bundle carries blob %s which this snapshot does not declare", name)
|
||
}
|
||
if part.Bytes != header.Size {
|
||
return Manifest{}, fmt.Errorf("bundle blob %s size %d does not match manifest %d", name, header.Size, part.Bytes)
|
||
}
|
||
if received[name] {
|
||
return Manifest{}, fmt.Errorf("bundle repeats blob %s", name)
|
||
}
|
||
if err := a.installBlob(reader, part, options); err != nil {
|
||
return Manifest{}, err
|
||
}
|
||
received[name] = true
|
||
default:
|
||
return Manifest{}, fmt.Errorf("bundle entry %q is not part of a snapshot bundle", header.Name)
|
||
}
|
||
}
|
||
if id == "" {
|
||
return Manifest{}, errors.New("bundle has no manifest")
|
||
}
|
||
for name := range expected {
|
||
if !received[name] {
|
||
return Manifest{}, fmt.Errorf("bundle is missing blob %s", name)
|
||
}
|
||
}
|
||
if err := syncRoot(a.blobs); err != nil {
|
||
return Manifest{}, err
|
||
}
|
||
if err := a.installManifest(id, manifestRaw); err != nil {
|
||
return Manifest{}, err
|
||
}
|
||
if err := syncRoot(a.versions); err != nil {
|
||
return Manifest{}, err
|
||
}
|
||
// 落位后用正式读取路径再验一遍:证明这份快照在本机真的可发布。
|
||
if _, err := a.Load(id); err != nil {
|
||
return Manifest{}, fmt.Errorf("imported snapshot does not verify: %w", err)
|
||
}
|
||
return manifest, nil
|
||
}
|
||
|
||
// installBlob 把一个 blob 从包里写到 blobs/<sha256>。
|
||
// 已存在时必须字节一致,否则拒绝——同名不同内容是身份冲突,不是"覆盖一下"。
|
||
func (a *Archive) installBlob(body io.Reader, part Part, options UnpackOptions) error {
|
||
if _, err := a.blobs.Lstat(part.SHA256); err == nil {
|
||
if verifyErr := a.verify(part); verifyErr == nil {
|
||
// 已有且一致:仍然读完包里的字节,否则 tar 流会错位。
|
||
_, err := io.CopyN(io.Discard, body, part.Bytes)
|
||
return err
|
||
}
|
||
if !options.RepairCorruptBlobs {
|
||
return fmt.Errorf("archive already holds different bytes for blob %s", part.SHA256)
|
||
}
|
||
// 名字就是内容摘要:同名而字节不符只可能是**本地损坏**。显式修复时删掉坏文件,
|
||
// 按包内字节原子重建;否则"本地坏块"永远无法自愈,页面会一直读成不可用。
|
||
if removeErr := a.blobs.Remove(part.SHA256); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
|
||
return fmt.Errorf("repair corrupt blob %s: %w", part.SHA256, removeErr)
|
||
}
|
||
} else if !errors.Is(err, os.ErrNotExist) {
|
||
return errors.New("archive blob lookup failed")
|
||
}
|
||
name, tmp, err := temporary(a.blobs)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer a.blobs.Remove(name)
|
||
hash := sha256.New()
|
||
written, err := io.Copy(io.MultiWriter(tmp, hash), io.LimitReader(body, part.Bytes+1))
|
||
if err != nil || written != part.Bytes || hex.EncodeToString(hash.Sum(nil)) != part.SHA256 {
|
||
tmp.Close()
|
||
return fmt.Errorf("bundle blob %s bytes do not match its digest", part.SHA256)
|
||
}
|
||
if err := tmp.Sync(); err != nil {
|
||
tmp.Close()
|
||
return err
|
||
}
|
||
if err := tmp.Close(); err != nil {
|
||
return err
|
||
}
|
||
if err := a.blobs.Link(name, part.SHA256); err != nil && !errors.Is(err, os.ErrExist) {
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// installManifest 落位版本清单(提交点)。已存在必须是同一份字节。
|
||
func (a *Archive) installManifest(id string, raw []byte) error {
|
||
if _, err := a.versions.Lstat(id + ".json"); err == nil {
|
||
existing, err := a.LoadManifest(id)
|
||
if err != nil {
|
||
return fmt.Errorf("archive already holds an unreadable version %s: %w", id, err)
|
||
}
|
||
_ = existing
|
||
return nil
|
||
} else if !errors.Is(err, os.ErrNotExist) {
|
||
return errors.New("archive version lookup failed")
|
||
}
|
||
name, tmp, err := temporary(a.versions)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer a.versions.Remove(name)
|
||
if _, err := tmp.Write(raw); err != nil {
|
||
tmp.Close()
|
||
return err
|
||
}
|
||
if err := tmp.Sync(); err != nil {
|
||
tmp.Close()
|
||
return err
|
||
}
|
||
if err := tmp.Close(); err != nil {
|
||
return err
|
||
}
|
||
if err := a.versions.Link(name, id+".json"); err != nil && !errors.Is(err, os.ErrExist) {
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// decodeBundleManifest 只接受"规范化字节"的清单:包里的清单必须与归档 Seal 时
|
||
// 写出的字节一致,否则即使摘要对得上,也可能是另一套字段排序/空值表达。
|
||
func decodeBundleManifest(raw []byte, id string) (Manifest, map[string]Part, error) {
|
||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||
decoder.DisallowUnknownFields()
|
||
var manifest Manifest
|
||
if err := decoder.Decode(&manifest); err != nil {
|
||
return Manifest{}, nil, errors.New("bundle manifest is not a valid snapshot manifest")
|
||
}
|
||
var extra any
|
||
if decoder.Decode(&extra) != io.EOF {
|
||
return Manifest{}, nil, errors.New("bundle manifest contains trailing data")
|
||
}
|
||
canonical, err := encodeManifest(manifest)
|
||
if err != nil {
|
||
return Manifest{}, nil, fmt.Errorf("bundle manifest is not publishable: %w", err)
|
||
}
|
||
if !bytes.Equal(canonical, raw) || digest(canonical) != id {
|
||
return Manifest{}, nil, errors.New("bundle manifest is not in canonical form")
|
||
}
|
||
expected := make(map[string]Part, len(manifest.Parts))
|
||
for _, part := range manifest.Parts {
|
||
if existing, ok := expected[part.SHA256]; ok {
|
||
if existing.Bytes != part.Bytes {
|
||
return Manifest{}, nil, errors.New("bundle manifest declares conflicting blob sizes")
|
||
}
|
||
continue
|
||
}
|
||
expected[part.SHA256] = part
|
||
}
|
||
return manifest, expected, nil
|
||
}
|
||
|
||
// DecodeManifest 校验一份"外部拿到的"版本清单字节:摘要必须等于给定快照 ID,
|
||
// 且必须是规范形式(与 Seal 写出的字节一致)。跨主机搬运时接收侧用它核对
|
||
// "这份清单是不是它自称的那份",而不是相信对方返回的字段。
|
||
func DecodeManifest(raw []byte, id string) (Manifest, error) {
|
||
if !validDigest(id) || digest(raw) != id {
|
||
return Manifest{}, errors.New("snapshot manifest digest mismatch")
|
||
}
|
||
manifest, _, err := decodeBundleManifest(raw, id)
|
||
if err != nil {
|
||
return Manifest{}, err
|
||
}
|
||
return manifest, nil
|
||
}
|