package pp import ( "bufio" "encoding/json" "fmt" "io" "sort" "knowledge-governance/internal/governance" ) type LegacyStatus string const ( StatusCompleted LegacyStatus = "completed" StatusContentAlias LegacyStatus = "content_alias" StatusOverLimit LegacyStatus = "over_limit" ) type Item struct { KBID string `json:"kb_id"` SourceURLHash string `json:"source_url_hash"` SourceIDs []string `json:"source_ids"` Bytes int64 `json:"bytes"` SHA256 string `json:"sha256"` Status LegacyStatus `json:"status"` KnowledgeID string `json:"knowledge_id"` CanonicalSourceURLHash string `json:"canonical_source_url_hash"` } type Snapshot struct { KBID string EventCount int UnresolvedCount int Items []Item } func FoldLedger(reader io.Reader) (Snapshot, error) { scanner := bufio.NewScanner(reader) scanner.Buffer(make([]byte, 64*1024), 1024*1024) terminal := make(map[string]Item) seen := make(map[string]struct{}) result := Snapshot{} for scanner.Scan() { result.EventCount++ var item Item if err := json.Unmarshal(scanner.Bytes(), &item); err != nil { return Snapshot{}, fmt.Errorf("decode ledger line %d: %w", result.EventCount, err) } if result.KBID == "" { result.KBID = item.KBID } else if item.KBID != result.KBID { return Snapshot{}, fmt.Errorf("ledger contains multiple knowledge bases at line %d", result.EventCount) } if item.SourceURLHash != "" { seen[item.SourceURLHash] = struct{}{} } if item.Status == StatusCompleted || (item.Status == StatusContentAlias && item.KnowledgeID != "") { if err := validateTerminalItem(item); err != nil { return Snapshot{}, fmt.Errorf("invalid terminal ledger record at line %d: %w", result.EventCount, err) } terminal[item.SourceURLHash] = item } } if err := scanner.Err(); err != nil { return Snapshot{}, fmt.Errorf("read ledger: %w", err) } result.Items = make([]Item, 0, len(terminal)) for _, item := range terminal { result.Items = append(result.Items, item) } result.UnresolvedCount = len(seen) - len(terminal) sort.Slice(result.Items, func(i, j int) bool { return result.Items[i].SourceURLHash < result.Items[j].SourceURLHash }) return result, nil } func validateTerminalItem(item Item) error { if item.KBID == "" { return fmt.Errorf("kb_id is required") } if !governance.ValidSHA256(item.SourceURLHash) { return fmt.Errorf("source_url_hash must be a canonical hash") } if len(item.SourceIDs) == 0 { return fmt.Errorf("source_ids are required") } for _, sourceID := range item.SourceIDs { if sourceID == "" { return fmt.Errorf("source_ids cannot contain an empty value") } } if item.Bytes < 0 { return fmt.Errorf("bytes cannot be negative") } if item.SHA256 != "" && !governance.ValidSHA256(item.SHA256) { return fmt.Errorf("sha256 must be empty or a canonical hash") } if item.KnowledgeID == "" { return fmt.Errorf("knowledge_id is required") } if item.Status == StatusContentAlias && !governance.ValidSHA256(item.CanonicalSourceURLHash) { return fmt.Errorf("canonical_source_url_hash must be a canonical hash") } return nil } func ValidateSnapshot(snapshot Snapshot) error { if snapshot.UnresolvedCount != 0 { return fmt.Errorf("snapshot contains %d unresolved source keys", snapshot.UnresolvedCount) } bySourceHash := make(map[string]Item, len(snapshot.Items)) for _, item := range snapshot.Items { if item.Status == StatusOverLimit { if item.Bytes <= LegacyMaxBytes { return fmt.Errorf("over-limit item does not exceed the legacy size limit") } bySourceHash[item.SourceURLHash] = item continue } if !governance.ValidSHA256(item.SHA256) { return fmt.Errorf("snapshot item is missing a verified SHA-256") } bySourceHash[item.SourceURLHash] = item } for _, item := range snapshot.Items { if item.Status != StatusContentAlias { continue } canonical, ok := bySourceHash[item.CanonicalSourceURLHash] if !ok || canonical.Status != StatusCompleted { return fmt.Errorf("alias does not reference a canonical completed item") } if item.SHA256 != canonical.SHA256 || item.KnowledgeID != canonical.KnowledgeID { return fmt.Errorf("alias metadata does not match canonical completed item") } } return nil }