knowledge-governance/internal/connectors/pp/ledger.go

package pp

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"sort"

	"knowledge-governance/internal/governance"
)

type LegacyStatus string

const (
	StatusCompleted    LegacyStatus = "completed"
	StatusContentAlias LegacyStatus = "content_alias"
	StatusOverLimit    LegacyStatus = "over_limit"
)

type Item struct {
	KBID                   string       `json:"kb_id"`
	SourceURLHash          string       `json:"source_url_hash"`
	SourceIDs              []string     `json:"source_ids"`
	Bytes                  int64        `json:"bytes"`
	SHA256                 string       `json:"sha256"`
	Status                 LegacyStatus `json:"status"`
	KnowledgeID            string       `json:"knowledge_id"`
	CanonicalSourceURLHash string       `json:"canonical_source_url_hash"`
}

type Snapshot struct {
	KBID            string
	EventCount      int
	UnresolvedCount int
	Items           []Item
}

func FoldLedger(reader io.Reader) (Snapshot, error) {
	scanner := bufio.NewScanner(reader)
	scanner.Buffer(make([]byte, 64*1024), 1024*1024)
	terminal := make(map[string]Item)
	seen := make(map[string]struct{})
	result := Snapshot{}
	for scanner.Scan() {
		result.EventCount++
		var item Item
		if err := json.Unmarshal(scanner.Bytes(), &item); err != nil {
			return Snapshot{}, fmt.Errorf("decode ledger line %d: %w", result.EventCount, err)
		}
		if result.KBID == "" {
			result.KBID = item.KBID
		} else if item.KBID != result.KBID {
			return Snapshot{}, fmt.Errorf("ledger contains multiple knowledge bases at line %d", result.EventCount)
		}
		if item.SourceURLHash != "" {
			seen[item.SourceURLHash] = struct{}{}
		}
		if item.Status == StatusCompleted || (item.Status == StatusContentAlias && item.KnowledgeID != "") {
			if err := validateTerminalItem(item); err != nil {
				return Snapshot{}, fmt.Errorf("invalid terminal ledger record at line %d: %w", result.EventCount, err)
			}
			terminal[item.SourceURLHash] = item
		}
	}
	if err := scanner.Err(); err != nil {
		return Snapshot{}, fmt.Errorf("read ledger: %w", err)
	}
	result.Items = make([]Item, 0, len(terminal))
	for _, item := range terminal {
		result.Items = append(result.Items, item)
	}
	result.UnresolvedCount = len(seen) - len(terminal)
	sort.Slice(result.Items, func(i, j int) bool {
		return result.Items[i].SourceURLHash < result.Items[j].SourceURLHash
	})
	return result, nil
}

func validateTerminalItem(item Item) error {
	if item.KBID == "" {
		return fmt.Errorf("kb_id is required")
	}
	if !governance.ValidSHA256(item.SourceURLHash) {
		return fmt.Errorf("source_url_hash must be a canonical hash")
	}
	if len(item.SourceIDs) == 0 {
		return fmt.Errorf("source_ids are required")
	}
	for _, sourceID := range item.SourceIDs {
		if sourceID == "" {
			return fmt.Errorf("source_ids cannot contain an empty value")
		}
	}
	if item.Bytes < 0 {
		return fmt.Errorf("bytes cannot be negative")
	}
	if item.SHA256 != "" && !governance.ValidSHA256(item.SHA256) {
		return fmt.Errorf("sha256 must be empty or a canonical hash")
	}
	if item.KnowledgeID == "" {
		return fmt.Errorf("knowledge_id is required")
	}
	if item.Status == StatusContentAlias && !governance.ValidSHA256(item.CanonicalSourceURLHash) {
		return fmt.Errorf("canonical_source_url_hash must be a canonical hash")
	}
	return nil
}

func ValidateSnapshot(snapshot Snapshot) error {
	if snapshot.UnresolvedCount != 0 {
		return fmt.Errorf("snapshot contains %d unresolved source keys", snapshot.UnresolvedCount)
	}
	bySourceHash := make(map[string]Item, len(snapshot.Items))
	for _, item := range snapshot.Items {
		if item.Status == StatusOverLimit {
			if item.Bytes <= LegacyMaxBytes {
				return fmt.Errorf("over-limit item does not exceed the legacy size limit")
			}
			bySourceHash[item.SourceURLHash] = item
			continue
		}
		if !governance.ValidSHA256(item.SHA256) {
			return fmt.Errorf("snapshot item is missing a verified SHA-256")
		}
		bySourceHash[item.SourceURLHash] = item
	}
	for _, item := range snapshot.Items {
		if item.Status != StatusContentAlias {
			continue
		}
		canonical, ok := bySourceHash[item.CanonicalSourceURLHash]
		if !ok || canonical.Status != StatusCompleted {
			return fmt.Errorf("alias does not reference a canonical completed item")
		}
		if item.SHA256 != canonical.SHA256 || item.KnowledgeID != canonical.KnowledgeID {
			return fmt.Errorf("alias metadata does not match canonical completed item")
		}
	}
	return nil
}