package librarysnapshot import ( "crypto/sha256" "encoding/hex" "errors" "io" "math" "os" "slices" "sort" "strings" "syscall" ) type Usage struct { VersionID string `json:"version_id"` LogicalBytes int64 `json:"logical_bytes"` UniqueBytes int64 `json:"unique_bytes"` SharedBytes int64 `json:"shared_bytes"` ExclusiveBytes int64 `json:"exclusive_bytes"` } // OpenPart provides archive-owned recovery bytes, never a source-side locator. // Verify and rewind the same descriptor, so path replacement cannot swap the // verified file for an unchecked dependency between these two operations. func (a *Archive) OpenPart(p Part) (io.ReadCloser, error) { if !validDigest(p.SHA256) || p.Bytes < 0 || p.Bytes > a.maxBlobBytes { return nil, errors.New("invalid recovery blob") } f, err := regular(a.blobs, p.SHA256, a.maxBlobBytes) if err != nil { return nil, err } h := sha256.New() n, err := io.Copy(h, io.LimitReader(f, a.maxBlobBytes+1)) if err != nil || n != p.Bytes || hex.EncodeToString(h.Sum(nil)) != p.SHA256 { f.Close() return nil, errors.New("corrupt recovery blob") } if _, err = f.Seek(0, io.SeekStart); err != nil { f.Close() return nil, err } return f, nil } // Usage reports content bytes per version and is **verified**: every referenced // blob is re-read and re-hashed through Load. Execution paths (sync/restore) and // the operator CLI use it, because acting on a version must not trust the // manifest's own claim about bytes it did not check. func (a *Archive) Usage() ([]Usage, error) { return a.usage(a.Load) } // UsageFromManifests is the read-only listing counterpart: the same accounting, // but it validates only each manifest's own digest/structure instead of // re-hashing every blob. // // Why it exists: a listing does not execute anything, and re-hashing every blob // of every version turned a 15 ms inventory into a 13.8 s timeout at 79 versions // (measured on the candidate archive). Keep the numbers real: they are recomputed // from the current inventory on every call, under the same "inventory must not // change while reading" guard, and an unreadable/invalid manifest still fails the // whole read. What this does *not* prove is that the referenced bytes are intact — // Load/OpenPart still decide that, and callers must not report this as // "integrity verified". func (a *Archive) UsageFromManifests() ([]Usage, error) { return a.usage(a.LoadManifest) } func (a *Archive) usage(load func(string) (Manifest, error)) ([]Usage, error) { ids, err := a.versionIDs() if err != nil { return nil, err } type inventory struct { usage Usage blobs map[string]int64 } inventories := make([]inventory, 0, len(ids)) refs := map[string]int{} for _, id := range ids { m, err := load(id) if err != nil { return nil, err } v := inventory{usage: Usage{VersionID: id}, blobs: map[string]int64{}} for _, p := range m.Parts { if v.usage.LogicalBytes > math.MaxInt64-p.Bytes { return nil, errors.New("snapshot usage overflow") } v.usage.LogicalBytes += p.Bytes v.blobs[p.SHA256] = p.Bytes } for sha := range v.blobs { refs[sha]++ } inventories = append(inventories, v) } after, err := a.versionIDs() if err != nil { return nil, err } if !slices.Equal(ids, after) { return nil, errors.New("snapshot inventory changed; retry usage read") } result := make([]Usage, 0, len(inventories)) for _, v := range inventories { for sha, size := range v.blobs { v.usage.UniqueBytes += size if refs[sha] > 1 { v.usage.SharedBytes += size } else { v.usage.ExclusiveBytes += size } } result = append(result, v.usage) } return result, nil } // ContentUsage is the archive-wide accounting of *content* bytes. // // It exists because per-version Usage() deduplicates only within one version: a // blob shared by two versions is counted in both UniqueBytes, so summing those // numbers over-reports the archive. ContentBytes here is deduplicated across // every sealed version. // // It is still not filesystem occupancy: manifests, unreferenced blobs left by an // interrupted Put and filesystem allocation (block rounding, metadata) are // reported separately, and none of these numbers are reclaimable space. Cleanup // needs retention/protection and live-job references from the coordinator. type ContentUsage struct { Versions int `json:"versions"` ReferencedBlobs int `json:"referenced_blobs"` ContentBytes int64 `json:"content_bytes"` UnreferencedBlobs int `json:"unreferenced_blobs"` UnreferencedBytes int64 `json:"unreferenced_bytes"` BlobFileBytes int64 `json:"blob_file_bytes"` BlobAllocatedBytes int64 `json:"blob_allocated_bytes"` ManifestFileBytes int64 `json:"manifest_file_bytes"` ManifestAllocatedBytes int64 `json:"manifest_allocated_bytes"` } // ContentUsage walks the sealed versions and the blob directory once, with the // same "inventory must not change under us" guard as Usage. func (a *Archive) ContentUsage() (ContentUsage, error) { var usage ContentUsage ids, err := a.versionIDs() if err != nil { return usage, err } referenced := map[string]int64{} for _, id := range ids { // 盘点只读清单:逐版重算 blob 会把一次统计放大到全部版本字节(实测 13.8s)。 m, err := a.LoadManifest(id) if err != nil { return usage, err } for _, p := range m.Parts { referenced[p.SHA256] = p.Bytes } } after, err := a.versionIDs() if err != nil { return usage, err } if !slices.Equal(ids, after) { return usage, errors.New("snapshot inventory changed; retry usage read") } usage.Versions = len(ids) usage.ReferencedBlobs = len(referenced) for _, size := range referenced { if usage.ContentBytes > math.MaxInt64-size { return usage, errors.New("snapshot usage overflow") } usage.ContentBytes += size } blobs, err := directoryBytes(a.blobs, func(name string) error { if !validDigest(name) { return errors.New("unexpected archive blob entry") } return nil }) if err != nil { return usage, err } usage.BlobFileBytes, usage.BlobAllocatedBytes = blobs.fileBytes, blobs.allocatedBytes usage.UnreferencedBlobs = blobs.files - len(referenced) usage.UnreferencedBytes = blobs.fileBytes for _, size := range referenced { usage.UnreferencedBytes -= size } if usage.UnreferencedBytes < 0 || usage.UnreferencedBlobs < 0 { return usage, errors.New("archive blob inventory is inconsistent") } manifests, err := directoryBytes(a.versions, func(name string) error { if strings.HasPrefix(name, ".pending-") { return nil } if !strings.HasSuffix(name, ".json") { return errors.New("unexpected archive version entry") } return nil }) if err != nil { return usage, err } usage.ManifestFileBytes, usage.ManifestAllocatedBytes = manifests.fileBytes, manifests.allocatedBytes return usage, nil } type directoryUsage struct { files int fileBytes int64 allocatedBytes int64 } // directoryBytes reports apparent size and allocated size (st_blocks) so callers // can tell content size from what the filesystem actually reserved. func directoryBytes(root *os.Root, validate func(string) error) (directoryUsage, error) { var usage directoryUsage dir, err := root.Open(".") if err != nil { return usage, err } defer dir.Close() entries, err := dir.ReadDir(-1) if err != nil { return usage, err } for _, entry := range entries { if err := validate(entry.Name()); err != nil { return usage, err } info, err := entry.Info() if err != nil { return usage, err } if !info.Mode().IsRegular() { return usage, errors.New("archive entry is not a regular file") } usage.files++ usage.fileBytes += info.Size() if stat, ok := info.Sys().(*syscall.Stat_t); ok { usage.allocatedBytes += stat.Blocks * 512 } } return usage, nil } func (a *Archive) versionIDs() ([]string, error) { f, err := a.versions.Open(".") if err != nil { return nil, err } defer f.Close() entries, err := f.ReadDir(-1) if err != nil { return nil, err } ids := make([]string, 0, len(entries)) for _, entry := range entries { if strings.HasPrefix(entry.Name(), ".pending-") { continue } name := entry.Name() id := strings.TrimSuffix(name, ".json") if !strings.HasSuffix(name, ".json") || !validDigest(id) || entry.IsDir() { return nil, errors.New("unexpected archive version entry") } ids = append(ids, id) } sort.Strings(ids) return ids, nil }