package v071 import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "io" "sort" "strings" ) type ObjectReader interface { Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) } type ObjectExporterLimits struct { MaxObjectBytes int64 MaxTotalBytes int64 } type AssetObject struct { ResourceID string Handle string Provider string PhysicalPath string Size int64 SHA256 string Data []byte } type ObjectSnapshot struct { Objects []AssetObject TotalBytes int64 SHA256 string } type ObjectExporter struct { reader ObjectReader limits ObjectExporterLimits } func NewObjectExporter(reader ObjectReader, limits ObjectExporterLimits) (*ObjectExporter, error) { if reader == nil { return nil, errors.New("object reader is required") } if limits.MaxObjectBytes == 0 { limits.MaxObjectBytes = 100 * 1024 * 1024 } if limits.MaxTotalBytes == 0 { limits.MaxTotalBytes = 1024 * 1024 * 1024 } if limits.MaxObjectBytes < 1 || limits.MaxTotalBytes < limits.MaxObjectBytes { return nil, errors.New("invalid object export limits") } return &ObjectExporter{reader: reader, limits: limits}, nil } func (exporter *ObjectExporter) Export(ctx context.Context, postgresSnapshot PostgresSnapshot) (ObjectSnapshot, error) { resources := postgresSnapshot.Table("resources") objects := make([]AssetObject, 0, len(resources.Rows)) var total int64 for _, raw := range resources.Rows { var resource struct { ID string `json:"id"` Handle string `json:"handle"` Provider string `json:"provider"` PhysicalPath string `json:"physical_path"` Size int64 `json:"size"` ContentHash string `json:"content_hash"` } if json.Unmarshal(raw, &resource) != nil || !validSearchIdentifier(resource.ID) || len(resource.Handle) != 22 || !validObjectProvider(resource.Provider) || !validObjectKey(resource.PhysicalPath) || resource.Size < 0 || resource.Size > exporter.limits.MaxObjectBytes || (resource.ContentHash != "" && !validLowerSHA256(resource.ContentHash)) { return ObjectSnapshot{}, errors.New("invalid release resource record") } reader, err := exporter.reader.Open(ctx, resource.Provider, resource.PhysicalPath) if err != nil { return ObjectSnapshot{}, errors.New("read release object") } data, readErr := io.ReadAll(io.LimitReader(reader, exporter.limits.MaxObjectBytes+1)) closeErr := reader.Close() if readErr != nil || closeErr != nil || int64(len(data)) > exporter.limits.MaxObjectBytes { return ObjectSnapshot{}, errors.New("read release object") } if resource.Size != int64(len(data)) { return ObjectSnapshot{}, errors.New("release object size mismatch") } digest := sha256.Sum256(data) contentSHA := hex.EncodeToString(digest[:]) if resource.ContentHash != "" && resource.ContentHash != contentSHA { return ObjectSnapshot{}, errors.New("release object digest mismatch") } total += int64(len(data)) if total > exporter.limits.MaxTotalBytes { return ObjectSnapshot{}, errors.New("release object total exceeds limit") } objects = append(objects, AssetObject{ ResourceID: resource.ID, Handle: resource.Handle, Provider: resource.Provider, PhysicalPath: resource.PhysicalPath, Size: int64(len(data)), SHA256: contentSHA, Data: data, }) } sort.Slice(objects, func(left, right int) bool { return objects[left].Handle < objects[right].Handle }) return ObjectSnapshot{Objects: objects, TotalBytes: total, SHA256: objectSnapshotDigest(objects)}, nil } func objectSnapshotDigest(objects []AssetObject) string { inventory := make([]struct { Handle string `json:"handle"` Key string `json:"key"` Size int64 `json:"size"` SHA256 string `json:"sha256"` }, 0, len(objects)) for _, object := range objects { inventory = append(inventory, struct { Handle string `json:"handle"` Key string `json:"key"` Size int64 `json:"size"` SHA256 string `json:"sha256"` }{Handle: object.Handle, Key: object.PhysicalPath, Size: object.Size, SHA256: object.SHA256}) } encoded, _ := json.Marshal(inventory) digest := sha256.Sum256(encoded) return hex.EncodeToString(digest[:]) } func shaBytes(data []byte) string { digest := sha256.Sum256(data) return hex.EncodeToString(digest[:]) } func validObjectProvider(provider string) bool { switch provider { case "minio", "oss", "s3", "cos": return true default: return false } } func validObjectKey(value string) bool { if strings.HasPrefix(value, "storage://") { rest := strings.TrimPrefix(value, "storage://") parts := strings.SplitN(rest, "/", 2) return len(parts) == 2 && validSearchIdentifier(parts[0]) && validObjectLocator(parts[1]) } return validObjectLocator(value) } func validLowerSHA256(value string) bool { if len(value) != 64 { return false } for _, char := range value { if (char < '0' || char > '9') && (char < 'a' || char > 'f') { return false } } return true }