package librarysnapshot import ( "archive/tar" "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "sort" "syscall" ) // 跨主机搬运一份已封存快照。 // // 为什么不是"把 versions/ 和 blobs/ 目录 rsync 过去": // - 直接同步目录时,"版本清单先到、blob 还在路上"的瞬间看起来就是一份可用版本, // 下一个发布作业会读到缺字节的档然后把失败记在目标身上; // - 也不能只搬 blobs:清单之外的 blob 无法证明属于这份内容。 // // 所以搬运单位是"自描述包":清单先写、每个 blob 以 sha256 命名、下载侧边收边算, // 只有全部齐了才把版本清单落位。版本清单文件就是提交点(见 Install 的注释)。 const BundleFormat = "weknora-library-snapshot-bundle/v1" // 包内路径。清单带身份(文件名即快照 ID),blob 带内容身份(文件名即 sha256): // 接收侧不需要额外的锁文件或元数据就能判断"这是什么、字节对不对"。 const ( bundleManifestPrefix = "manifest/" bundleBlobPrefix = "blobs/" maxBundleEntries = 200000 ) // BundleInfo 是一次导出的结果(CLI 打印它,父据此核对哈希)。 type BundleInfo struct { Format string `json:"format"` Snapshot string `json:"snapshot"` Parts int `json:"parts"` Bytes int64 `json:"bytes"` // Blobs 是逐个 blob 的 sha256 列表,便于在两台机器上对拍而不传大文件。 Blobs []string `json:"blobs"` } // Pack 把一份已封存版本导出成单文件包(tar)。 // // 导出前先 Load:它逐 blob 重算 sha256,所以"包里的每个字节都在这台机器上验证过"。 // 导出后不删源,也不改归档。 func (a *Archive) Pack(id, outPath string) (BundleInfo, error) { if !validDigest(id) { return BundleInfo{}, errors.New("invalid snapshot ID") } if !filepath.IsAbs(outPath) || filepath.Clean(outPath) != outPath { return BundleInfo{}, errors.New("bundle path must be an absolute clean path") } manifest, err := a.Load(id) if err != nil { return BundleInfo{}, fmt.Errorf("snapshot is not verifiable on this host: %w", err) } manifestFile, err := regular(a.versions, id+".json", maxManifestBytes) if err != nil { return BundleInfo{}, err } manifestBytes, readErr := io.ReadAll(io.LimitReader(manifestFile, maxManifestBytes+1)) manifestFile.Close() if readErr != nil || int64(len(manifestBytes)) > maxManifestBytes || digest(manifestBytes) != id { return BundleInfo{}, errors.New("snapshot manifest changed while packing") } if _, err := os.Lstat(outPath); err == nil { return BundleInfo{}, errors.New("bundle already exists; refusing to overwrite") } else if !errors.Is(err, os.ErrNotExist) { return BundleInfo{}, errors.New("bundle path is not usable") } out, err := os.OpenFile(outPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) if err != nil { return BundleInfo{}, errors.New("create bundle") } cleanup := func(cause error) (BundleInfo, error) { out.Close() os.Remove(outPath) return BundleInfo{}, cause } writer := tar.NewWriter(out) writeEntry := func(name string, size int64, body io.Reader) error { header := &tar.Header{Name: name, Mode: 0600, Size: size, Typeflag: tar.TypeReg} if err := writer.WriteHeader(header); err != nil { return err } _, err := io.CopyN(writer, body, size) return err } // 清单必须第一个写:接收侧读到清单才知道后面应该有哪些 blob。 if err := writeEntry(bundleManifestPrefix+id+".json", int64(len(manifestBytes)), bytes.NewReader(manifestBytes)); err != nil { return cleanup(errors.New("write bundle manifest")) } info := BundleInfo{Format: BundleFormat, Snapshot: id, Parts: len(manifest.Parts)} blobs := make([]string, 0, len(manifest.Parts)) seen := map[string]bool{} for _, part := range manifest.Parts { if seen[part.SHA256] { continue } seen[part.SHA256] = true file, err := regular(a.blobs, part.SHA256, a.maxBlobBytes) if err != nil { return cleanup(fmt.Errorf("snapshot blob %s is not readable: %w", part.SHA256, err)) } writeErr := writeEntry(bundleBlobPrefix+part.SHA256, part.Bytes, file) file.Close() if writeErr != nil { return cleanup(errors.New("write bundle blob")) } info.Bytes += part.Bytes blobs = append(blobs, part.SHA256) } if err := writer.Close(); err != nil { return cleanup(errors.New("finalize bundle")) } if err := out.Sync(); err != nil { return cleanup(errors.New("sync bundle")) } if err := out.Close(); err != nil { return BundleInfo{}, errors.New("close bundle") } sort.Strings(blobs) info.Blobs = blobs return info, nil } // Unpack 校验一个包并把它装入本归档。返回值是这份快照自己的清单。 // // 提交点是"版本清单落位":blob 先按内容寻址写好并 fsync,最后一步才 Link // versions/.json。因此任何中断都只会留下不可见的 blob(可被下一次重放复用), // 不会留下"看起来存在但缺字节"的版本。 // // 幂等:同一份包重复导入成功;名字相同内容不同的 blob 或清单直接拒绝。 // UnpackOptions 控制"本地已有同名 blob 但字节不符"时的行为。 // // 默认拒绝(同 ID 异内容一律当身份冲突处理)。只有"从权威云端副本读回"这一条路径 // 显式打开修复:那时同名 blob 字节不符只能是本地损坏,拒绝会让这份版本永远不可读。 type UnpackOptions struct { RepairCorruptBlobs bool } func (a *Archive) Unpack(bundlePath, expectID string) (Manifest, error) { return a.UnpackWithOptions(bundlePath, expectID, UnpackOptions{}) } func (a *Archive) UnpackWithOptions(bundlePath, expectID string, options UnpackOptions) (Manifest, error) { if expectID != "" && !validDigest(expectID) { return Manifest{}, errors.New("invalid expected snapshot ID") } if !filepath.IsAbs(bundlePath) || filepath.Clean(bundlePath) != bundlePath { return Manifest{}, errors.New("bundle path must be an absolute clean path") } file, err := os.OpenFile(bundlePath, os.O_RDONLY|syscall.O_NOFOLLOW, 0) if err != nil { return Manifest{}, errors.New("open bundle") } defer file.Close() if info, statErr := file.Stat(); statErr != nil || !info.Mode().IsRegular() { return Manifest{}, errors.New("bundle must be a regular file") } reader := tar.NewReader(file) var ( id string manifest Manifest manifestRaw []byte expected map[string]Part received map[string]bool entryCount int ) for { header, err := reader.Next() if errors.Is(err, io.EOF) { break } if err != nil { return Manifest{}, errors.New("bundle is not a readable archive") } entryCount++ if entryCount > maxBundleEntries { return Manifest{}, errors.New("bundle has too many entries") } if header.Typeflag != tar.TypeReg { return Manifest{}, fmt.Errorf("bundle entry %q is not a regular file", header.Name) } switch { case len(header.Name) == len(bundleManifestPrefix)+len(".json")+64 && header.Name[:len(bundleManifestPrefix)] == bundleManifestPrefix: if id != "" { return Manifest{}, errors.New("bundle contains more than one manifest") } id = header.Name[len(bundleManifestPrefix) : len(header.Name)-len(".json")] if !validDigest(id) { return Manifest{}, errors.New("bundle manifest name is not a snapshot ID") } if expectID != "" && id != expectID { return Manifest{}, fmt.Errorf("bundle snapshot is %s, expected %s", id, expectID) } if header.Size <= 0 || header.Size > maxManifestBytes { return Manifest{}, errors.New("bundle manifest size is not allowed") } manifestRaw, err = io.ReadAll(io.LimitReader(reader, maxManifestBytes+1)) if err != nil || int64(len(manifestRaw)) != header.Size || digest(manifestRaw) != id { return Manifest{}, errors.New("bundle manifest digest does not match its ID") } manifest, expected, err = decodeBundleManifest(manifestRaw, id) if err != nil { return Manifest{}, err } received = map[string]bool{} case len(header.Name) == len(bundleBlobPrefix)+64 && header.Name[:len(bundleBlobPrefix)] == bundleBlobPrefix: if id == "" { return Manifest{}, errors.New("bundle blob appears before the manifest") } name := header.Name[len(bundleBlobPrefix):] if !validDigest(name) { return Manifest{}, errors.New("bundle blob name is not a sha256 digest") } part, ok := expected[name] if !ok { return Manifest{}, fmt.Errorf("bundle carries blob %s which this snapshot does not declare", name) } if part.Bytes != header.Size { return Manifest{}, fmt.Errorf("bundle blob %s size %d does not match manifest %d", name, header.Size, part.Bytes) } if received[name] { return Manifest{}, fmt.Errorf("bundle repeats blob %s", name) } if err := a.installBlob(reader, part, options); err != nil { return Manifest{}, err } received[name] = true default: return Manifest{}, fmt.Errorf("bundle entry %q is not part of a snapshot bundle", header.Name) } } if id == "" { return Manifest{}, errors.New("bundle has no manifest") } for name := range expected { if !received[name] { return Manifest{}, fmt.Errorf("bundle is missing blob %s", name) } } if err := syncRoot(a.blobs); err != nil { return Manifest{}, err } if err := a.installManifest(id, manifestRaw); err != nil { return Manifest{}, err } if err := syncRoot(a.versions); err != nil { return Manifest{}, err } // 落位后用正式读取路径再验一遍:证明这份快照在本机真的可发布。 if _, err := a.Load(id); err != nil { return Manifest{}, fmt.Errorf("imported snapshot does not verify: %w", err) } return manifest, nil } // installBlob 把一个 blob 从包里写到 blobs/。 // 已存在时必须字节一致,否则拒绝——同名不同内容是身份冲突,不是"覆盖一下"。 func (a *Archive) installBlob(body io.Reader, part Part, options UnpackOptions) error { if _, err := a.blobs.Lstat(part.SHA256); err == nil { if verifyErr := a.verify(part); verifyErr == nil { // 已有且一致:仍然读完包里的字节,否则 tar 流会错位。 _, err := io.CopyN(io.Discard, body, part.Bytes) return err } if !options.RepairCorruptBlobs { return fmt.Errorf("archive already holds different bytes for blob %s", part.SHA256) } // 名字就是内容摘要:同名而字节不符只可能是**本地损坏**。显式修复时删掉坏文件, // 按包内字节原子重建;否则"本地坏块"永远无法自愈,页面会一直读成不可用。 if removeErr := a.blobs.Remove(part.SHA256); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { return fmt.Errorf("repair corrupt blob %s: %w", part.SHA256, removeErr) } } else if !errors.Is(err, os.ErrNotExist) { return errors.New("archive blob lookup failed") } name, tmp, err := temporary(a.blobs) if err != nil { return err } defer a.blobs.Remove(name) hash := sha256.New() written, err := io.Copy(io.MultiWriter(tmp, hash), io.LimitReader(body, part.Bytes+1)) if err != nil || written != part.Bytes || hex.EncodeToString(hash.Sum(nil)) != part.SHA256 { tmp.Close() return fmt.Errorf("bundle blob %s bytes do not match its digest", part.SHA256) } if err := tmp.Sync(); err != nil { tmp.Close() return err } if err := tmp.Close(); err != nil { return err } if err := a.blobs.Link(name, part.SHA256); err != nil && !errors.Is(err, os.ErrExist) { return err } return nil } // installManifest 落位版本清单(提交点)。已存在必须是同一份字节。 func (a *Archive) installManifest(id string, raw []byte) error { if _, err := a.versions.Lstat(id + ".json"); err == nil { existing, err := a.LoadManifest(id) if err != nil { return fmt.Errorf("archive already holds an unreadable version %s: %w", id, err) } _ = existing return nil } else if !errors.Is(err, os.ErrNotExist) { return errors.New("archive version lookup failed") } name, tmp, err := temporary(a.versions) if err != nil { return err } defer a.versions.Remove(name) if _, err := tmp.Write(raw); err != nil { tmp.Close() return err } if err := tmp.Sync(); err != nil { tmp.Close() return err } if err := tmp.Close(); err != nil { return err } if err := a.versions.Link(name, id+".json"); err != nil && !errors.Is(err, os.ErrExist) { return err } return nil } // decodeBundleManifest 只接受"规范化字节"的清单:包里的清单必须与归档 Seal 时 // 写出的字节一致,否则即使摘要对得上,也可能是另一套字段排序/空值表达。 func decodeBundleManifest(raw []byte, id string) (Manifest, map[string]Part, error) { decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.DisallowUnknownFields() var manifest Manifest if err := decoder.Decode(&manifest); err != nil { return Manifest{}, nil, errors.New("bundle manifest is not a valid snapshot manifest") } var extra any if decoder.Decode(&extra) != io.EOF { return Manifest{}, nil, errors.New("bundle manifest contains trailing data") } canonical, err := encodeManifest(manifest) if err != nil { return Manifest{}, nil, fmt.Errorf("bundle manifest is not publishable: %w", err) } if !bytes.Equal(canonical, raw) || digest(canonical) != id { return Manifest{}, nil, errors.New("bundle manifest is not in canonical form") } expected := make(map[string]Part, len(manifest.Parts)) for _, part := range manifest.Parts { if existing, ok := expected[part.SHA256]; ok { if existing.Bytes != part.Bytes { return Manifest{}, nil, errors.New("bundle manifest declares conflicting blob sizes") } continue } expected[part.SHA256] = part } return manifest, expected, nil } // DecodeManifest 校验一份"外部拿到的"版本清单字节:摘要必须等于给定快照 ID, // 且必须是规范形式(与 Seal 写出的字节一致)。跨主机搬运时接收侧用它核对 // "这份清单是不是它自称的那份",而不是相信对方返回的字段。 func DecodeManifest(raw []byte, id string) (Manifest, error) { if !validDigest(id) || digest(raw) != id { return Manifest{}, errors.New("snapshot manifest digest mismatch") } manifest, _, err := decodeBundleManifest(raw, id) if err != nil { return Manifest{}, err } return manifest, nil }