Files
knowledge-governance/internal/librarysnapshot/usage.go
T
姚章浩 c5e033dd0e
build / build (push) Failing after 15s
fix(library): 幂等重放、跨版本去重用量与契约复核
- 同作业号重跑读持久日志+已封存清单按首次请求重放(source_read=false,
  writers_stopped=false),换范围/绑定库或清单校验失败即拒绝
- 归档用量按跨版本去重统计内容字节,清单/未引用 blob/实际分配单列,
  不再用各版本 unique 之和冒充归档占用
- 冻结窗口内复核来源契约,不一致即中止;拒绝组外同镜像运行容器
- 补齐整库只读接口的 OpenAPI 契约
- 按用户确认取消 A06:文档/页面/接口不再把效果测试列为封版门槛
2026-10-28 12:01:09 +08:00

256 lines
7.2 KiB
Go

package librarysnapshot
import (
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"math"
"os"
"slices"
"sort"
"strings"
"syscall"
)
type Usage struct {
VersionID string `json:"version_id"`
LogicalBytes int64 `json:"logical_bytes"`
UniqueBytes int64 `json:"unique_bytes"`
SharedBytes int64 `json:"shared_bytes"`
ExclusiveBytes int64 `json:"exclusive_bytes"`
}
// OpenPart provides archive-owned recovery bytes, never a source-side locator.
// Verify and rewind the same descriptor, so path replacement cannot swap the
// verified file for an unchecked dependency between these two operations.
func (a *Archive) OpenPart(p Part) (io.ReadCloser, error) {
if !validDigest(p.SHA256) || p.Bytes < 0 || p.Bytes > a.maxBlobBytes {
return nil, errors.New("invalid recovery blob")
}
f, err := regular(a.blobs, p.SHA256, a.maxBlobBytes)
if err != nil {
return nil, err
}
h := sha256.New()
n, err := io.Copy(h, io.LimitReader(f, a.maxBlobBytes+1))
if err != nil || n != p.Bytes || hex.EncodeToString(h.Sum(nil)) != p.SHA256 {
f.Close()
return nil, errors.New("corrupt recovery blob")
}
if _, err = f.Seek(0, io.SeekStart); err != nil {
f.Close()
return nil, err
}
return f, nil
}
// Usage reports content bytes, not filesystem allocation or reclaimed space.
// It is deliberately read-only: protection/retention and live-job references
// must be acquired from the coordinator before any future cleanup operation.
func (a *Archive) Usage() ([]Usage, error) {
ids, err := a.versionIDs()
if err != nil {
return nil, err
}
type inventory struct {
usage Usage
blobs map[string]int64
}
inventories := make([]inventory, 0, len(ids))
refs := map[string]int{}
for _, id := range ids {
m, err := a.Load(id)
if err != nil {
return nil, err
}
v := inventory{usage: Usage{VersionID: id}, blobs: map[string]int64{}}
for _, p := range m.Parts {
if v.usage.LogicalBytes > math.MaxInt64-p.Bytes {
return nil, errors.New("snapshot usage overflow")
}
v.usage.LogicalBytes += p.Bytes
v.blobs[p.SHA256] = p.Bytes
}
for sha := range v.blobs {
refs[sha]++
}
inventories = append(inventories, v)
}
after, err := a.versionIDs()
if err != nil {
return nil, err
}
if !slices.Equal(ids, after) {
return nil, errors.New("snapshot inventory changed; retry usage read")
}
result := make([]Usage, 0, len(inventories))
for _, v := range inventories {
for sha, size := range v.blobs {
v.usage.UniqueBytes += size
if refs[sha] > 1 {
v.usage.SharedBytes += size
} else {
v.usage.ExclusiveBytes += size
}
}
result = append(result, v.usage)
}
return result, nil
}
// ContentUsage is the archive-wide accounting of *content* bytes.
//
// It exists because per-version Usage() deduplicates only within one version: a
// blob shared by two versions is counted in both UniqueBytes, so summing those
// numbers over-reports the archive. ContentBytes here is deduplicated across
// every sealed version.
//
// It is still not filesystem occupancy: manifests, unreferenced blobs left by an
// interrupted Put and filesystem allocation (block rounding, metadata) are
// reported separately, and none of these numbers are reclaimable space. Cleanup
// needs retention/protection and live-job references from the coordinator.
type ContentUsage struct {
Versions int `json:"versions"`
ReferencedBlobs int `json:"referenced_blobs"`
ContentBytes int64 `json:"content_bytes"`
UnreferencedBlobs int `json:"unreferenced_blobs"`
UnreferencedBytes int64 `json:"unreferenced_bytes"`
BlobFileBytes int64 `json:"blob_file_bytes"`
BlobAllocatedBytes int64 `json:"blob_allocated_bytes"`
ManifestFileBytes int64 `json:"manifest_file_bytes"`
ManifestAllocatedBytes int64 `json:"manifest_allocated_bytes"`
}
// ContentUsage walks the sealed versions and the blob directory once, with the
// same "inventory must not change under us" guard as Usage.
func (a *Archive) ContentUsage() (ContentUsage, error) {
var usage ContentUsage
ids, err := a.versionIDs()
if err != nil {
return usage, err
}
referenced := map[string]int64{}
for _, id := range ids {
m, err := a.Load(id)
if err != nil {
return usage, err
}
for _, p := range m.Parts {
referenced[p.SHA256] = p.Bytes
}
}
after, err := a.versionIDs()
if err != nil {
return usage, err
}
if !slices.Equal(ids, after) {
return usage, errors.New("snapshot inventory changed; retry usage read")
}
usage.Versions = len(ids)
usage.ReferencedBlobs = len(referenced)
for _, size := range referenced {
if usage.ContentBytes > math.MaxInt64-size {
return usage, errors.New("snapshot usage overflow")
}
usage.ContentBytes += size
}
blobs, err := directoryBytes(a.blobs, func(name string) error {
if !validDigest(name) {
return errors.New("unexpected archive blob entry")
}
return nil
})
if err != nil {
return usage, err
}
usage.BlobFileBytes, usage.BlobAllocatedBytes = blobs.fileBytes, blobs.allocatedBytes
usage.UnreferencedBlobs = blobs.files - len(referenced)
usage.UnreferencedBytes = blobs.fileBytes
for _, size := range referenced {
usage.UnreferencedBytes -= size
}
if usage.UnreferencedBytes < 0 || usage.UnreferencedBlobs < 0 {
return usage, errors.New("archive blob inventory is inconsistent")
}
manifests, err := directoryBytes(a.versions, func(name string) error {
if strings.HasPrefix(name, ".pending-") {
return nil
}
if !strings.HasSuffix(name, ".json") {
return errors.New("unexpected archive version entry")
}
return nil
})
if err != nil {
return usage, err
}
usage.ManifestFileBytes, usage.ManifestAllocatedBytes = manifests.fileBytes, manifests.allocatedBytes
return usage, nil
}
type directoryUsage struct {
files int
fileBytes int64
allocatedBytes int64
}
// directoryBytes reports apparent size and allocated size (st_blocks) so callers
// can tell content size from what the filesystem actually reserved.
func directoryBytes(root *os.Root, validate func(string) error) (directoryUsage, error) {
var usage directoryUsage
dir, err := root.Open(".")
if err != nil {
return usage, err
}
defer dir.Close()
entries, err := dir.ReadDir(-1)
if err != nil {
return usage, err
}
for _, entry := range entries {
if err := validate(entry.Name()); err != nil {
return usage, err
}
info, err := entry.Info()
if err != nil {
return usage, err
}
if !info.Mode().IsRegular() {
return usage, errors.New("archive entry is not a regular file")
}
usage.files++
usage.fileBytes += info.Size()
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
usage.allocatedBytes += stat.Blocks * 512
}
}
return usage, nil
}
func (a *Archive) versionIDs() ([]string, error) {
f, err := a.versions.Open(".")
if err != nil {
return nil, err
}
defer f.Close()
entries, err := f.ReadDir(-1)
if err != nil {
return nil, err
}
ids := make([]string, 0, len(entries))
for _, entry := range entries {
if strings.HasPrefix(entry.Name(), ".pending-") {
continue
}
name := entry.Name()
id := strings.TrimSuffix(name, ".json")
if !strings.HasSuffix(name, ".json") || !validDigest(id) || entry.IsDir() {
return nil, errors.New("unexpected archive version entry")
}
ids = append(ids, id)
}
sort.Strings(ids)
return ids, nil
}