knowledge-governance/internal/librarysnapshot/usage.go

package librarysnapshot

import (
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"io"
	"math"
	"os"
	"slices"
	"sort"
	"strings"
	"syscall"
)

type Usage struct {
	VersionID      string `json:"version_id"`
	LogicalBytes   int64  `json:"logical_bytes"`
	UniqueBytes    int64  `json:"unique_bytes"`
	SharedBytes    int64  `json:"shared_bytes"`
	ExclusiveBytes int64  `json:"exclusive_bytes"`
}

// OpenPart provides archive-owned recovery bytes, never a source-side locator.
// Verify and rewind the same descriptor, so path replacement cannot swap the
// verified file for an unchecked dependency between these two operations.
func (a *Archive) OpenPart(p Part) (io.ReadCloser, error) {
	if !validDigest(p.SHA256) || p.Bytes < 0 || p.Bytes > a.maxBlobBytes {
		return nil, errors.New("invalid recovery blob")
	}
	f, err := regular(a.blobs, p.SHA256, a.maxBlobBytes)
	if err != nil {
		return nil, err
	}
	h := sha256.New()
	n, err := io.Copy(h, io.LimitReader(f, a.maxBlobBytes+1))
	if err != nil || n != p.Bytes || hex.EncodeToString(h.Sum(nil)) != p.SHA256 {
		f.Close()
		return nil, errors.New("corrupt recovery blob")
	}
	if _, err = f.Seek(0, io.SeekStart); err != nil {
		f.Close()
		return nil, err
	}
	return f, nil
}

// Usage reports content bytes per version and is **verified**: every referenced
// blob is re-read and re-hashed through Load. Execution paths (sync/restore) and
// the operator CLI use it, because acting on a version must not trust the
// manifest's own claim about bytes it did not check.
func (a *Archive) Usage() ([]Usage, error) { return a.usage(a.Load) }

// UsageFromManifests is the read-only listing counterpart: the same accounting,
// but it validates only each manifest's own digest/structure instead of
// re-hashing every blob.
//
// Why it exists: a listing does not execute anything, and re-hashing every blob
// of every version turned a 15 ms inventory into a 13.8 s timeout at 79 versions
// (measured on the candidate archive). Keep the numbers real: they are recomputed
// from the current inventory on every call, under the same "inventory must not
// change while reading" guard, and an unreadable/invalid manifest still fails the
// whole read. What this does *not* prove is that the referenced bytes are intact —
// Load/OpenPart still decide that, and callers must not report this as
// "integrity verified".
func (a *Archive) UsageFromManifests() ([]Usage, error) { return a.usage(a.LoadManifest) }

func (a *Archive) usage(load func(string) (Manifest, error)) ([]Usage, error) {
	ids, err := a.versionIDs()
	if err != nil {
		return nil, err
	}
	type inventory struct {
		usage Usage
		blobs map[string]int64
	}
	inventories := make([]inventory, 0, len(ids))
	refs := map[string]int{}
	for _, id := range ids {
		m, err := load(id)
		if err != nil {
			return nil, err
		}
		v := inventory{usage: Usage{VersionID: id}, blobs: map[string]int64{}}
		for _, p := range m.Parts {
			if v.usage.LogicalBytes > math.MaxInt64-p.Bytes {
				return nil, errors.New("snapshot usage overflow")
			}
			v.usage.LogicalBytes += p.Bytes
			v.blobs[p.SHA256] = p.Bytes
		}
		for sha := range v.blobs {
			refs[sha]++
		}
		inventories = append(inventories, v)
	}
	after, err := a.versionIDs()
	if err != nil {
		return nil, err
	}
	if !slices.Equal(ids, after) {
		return nil, errors.New("snapshot inventory changed; retry usage read")
	}
	result := make([]Usage, 0, len(inventories))
	for _, v := range inventories {
		for sha, size := range v.blobs {
			v.usage.UniqueBytes += size
			if refs[sha] > 1 {
				v.usage.SharedBytes += size
			} else {
				v.usage.ExclusiveBytes += size
			}
		}
		result = append(result, v.usage)
	}
	return result, nil
}

// ContentUsage is the archive-wide accounting of *content* bytes.
//
// It exists because per-version Usage() deduplicates only within one version: a
// blob shared by two versions is counted in both UniqueBytes, so summing those
// numbers over-reports the archive. ContentBytes here is deduplicated across
// every sealed version.
//
// It is still not filesystem occupancy: manifests, unreferenced blobs left by an
// interrupted Put and filesystem allocation (block rounding, metadata) are
// reported separately, and none of these numbers are reclaimable space. Cleanup
// needs retention/protection and live-job references from the coordinator.
type ContentUsage struct {
	Versions               int   `json:"versions"`
	ReferencedBlobs        int   `json:"referenced_blobs"`
	ContentBytes           int64 `json:"content_bytes"`
	UnreferencedBlobs      int   `json:"unreferenced_blobs"`
	UnreferencedBytes      int64 `json:"unreferenced_bytes"`
	BlobFileBytes          int64 `json:"blob_file_bytes"`
	BlobAllocatedBytes     int64 `json:"blob_allocated_bytes"`
	ManifestFileBytes      int64 `json:"manifest_file_bytes"`
	ManifestAllocatedBytes int64 `json:"manifest_allocated_bytes"`
}

// ContentUsage walks the sealed versions and the blob directory once, with the
// same "inventory must not change under us" guard as Usage.
func (a *Archive) ContentUsage() (ContentUsage, error) {
	var usage ContentUsage
	ids, err := a.versionIDs()
	if err != nil {
		return usage, err
	}
	referenced := map[string]int64{}
	for _, id := range ids {
		// 盘点只读清单：逐版重算 blob 会把一次统计放大到全部版本字节（实测 13.8s）。
		m, err := a.LoadManifest(id)
		if err != nil {
			return usage, err
		}
		for _, p := range m.Parts {
			referenced[p.SHA256] = p.Bytes
		}
	}
	after, err := a.versionIDs()
	if err != nil {
		return usage, err
	}
	if !slices.Equal(ids, after) {
		return usage, errors.New("snapshot inventory changed; retry usage read")
	}
	usage.Versions = len(ids)
	usage.ReferencedBlobs = len(referenced)
	for _, size := range referenced {
		if usage.ContentBytes > math.MaxInt64-size {
			return usage, errors.New("snapshot usage overflow")
		}
		usage.ContentBytes += size
	}
	blobs, err := directoryBytes(a.blobs, func(name string) error {
		if !validDigest(name) {
			return errors.New("unexpected archive blob entry")
		}
		return nil
	})
	if err != nil {
		return usage, err
	}
	usage.BlobFileBytes, usage.BlobAllocatedBytes = blobs.fileBytes, blobs.allocatedBytes
	usage.UnreferencedBlobs = blobs.files - len(referenced)
	usage.UnreferencedBytes = blobs.fileBytes
	for _, size := range referenced {
		usage.UnreferencedBytes -= size
	}
	if usage.UnreferencedBytes < 0 || usage.UnreferencedBlobs < 0 {
		return usage, errors.New("archive blob inventory is inconsistent")
	}
	manifests, err := directoryBytes(a.versions, func(name string) error {
		if strings.HasPrefix(name, ".pending-") {
			return nil
		}
		if !strings.HasSuffix(name, ".json") {
			return errors.New("unexpected archive version entry")
		}
		return nil
	})
	if err != nil {
		return usage, err
	}
	usage.ManifestFileBytes, usage.ManifestAllocatedBytes = manifests.fileBytes, manifests.allocatedBytes
	return usage, nil
}

type directoryUsage struct {
	files          int
	fileBytes      int64
	allocatedBytes int64
}

// directoryBytes reports apparent size and allocated size (st_blocks) so callers
// can tell content size from what the filesystem actually reserved.
func directoryBytes(root *os.Root, validate func(string) error) (directoryUsage, error) {
	var usage directoryUsage
	dir, err := root.Open(".")
	if err != nil {
		return usage, err
	}
	defer dir.Close()
	entries, err := dir.ReadDir(-1)
	if err != nil {
		return usage, err
	}
	for _, entry := range entries {
		if err := validate(entry.Name()); err != nil {
			return usage, err
		}
		info, err := entry.Info()
		if err != nil {
			return usage, err
		}
		if !info.Mode().IsRegular() {
			return usage, errors.New("archive entry is not a regular file")
		}
		usage.files++
		usage.fileBytes += info.Size()
		if stat, ok := info.Sys().(*syscall.Stat_t); ok {
			usage.allocatedBytes += stat.Blocks * 512
		}
	}
	return usage, nil
}

func (a *Archive) versionIDs() ([]string, error) {
	f, err := a.versions.Open(".")
	if err != nil {
		return nil, err
	}
	defer f.Close()
	entries, err := f.ReadDir(-1)
	if err != nil {
		return nil, err
	}
	ids := make([]string, 0, len(entries))
	for _, entry := range entries {
		if strings.HasPrefix(entry.Name(), ".pending-") {
			continue
		}
		name := entry.Name()
		id := strings.TrimSuffix(name, ".json")
		if !strings.HasSuffix(name, ".json") || !validDigest(id) || entry.IsDir() {
			return nil, errors.New("unexpected archive version entry")
		}
		ids = append(ids, id)
	}
	sort.Strings(ids)
	return ids, nil
}