package monitoring import ( "context" "fmt" "math" "sort" "time" "knowledge-governance/internal/governance" ) type windowJobOutcome struct { state string durationSeconds float64 // <0 when duration is unknowable } const ( metricsShortWindow = time.Hour metricsLongWindow = 24 * time.Hour metricsWorkerLease = 90 * time.Second // workers heartbeat every <= lease; fresher than stale leases counts active metricsSampleMargin = 2 * time.Second ) func computeJobWindowMetrics(outcomes []windowJobOutcome, windowSeconds int) governance.JobWindowMetrics { metrics := governance.JobWindowMetrics{WindowSeconds: windowSeconds} var durations []float64 for _, outcome := range outcomes { switch outcome.state { case "completed": metrics.JobsCompleted++ case "failed": metrics.JobsFailed++ } if outcome.durationSeconds >= 0 { durations = append(durations, outcome.durationSeconds) } } if finished := metrics.JobsCompleted + metrics.JobsFailed; finished > 0 { success := math.Round(float64(metrics.JobsCompleted)/float64(finished)*10000) / 100 failure := math.Round(float64(metrics.JobsFailed)/float64(finished)*10000) / 100 metrics.SuccessRate = &success metrics.FailureRate = &failure } if len(durations) > 0 { sort.Float64s(durations) var sum float64 for _, value := range durations { sum += value } average := math.Round(sum/float64(len(durations))*10) / 10 p50 := math.Round(percentile(durations, 0.50)*10) / 10 p95 := math.Round(percentile(durations, 0.95)*10) / 10 metrics.DurationAvgSeconds = &average metrics.DurationP50Seconds = &p50 metrics.DurationP95Seconds = &p95 } return metrics } // percentile uses nearest-rank so tiny samples never interpolate a fake value. func percentile(sorted []float64, fraction float64) float64 { rank := int(math.Ceil(fraction * float64(len(sorted)))) if rank < 1 { rank = 1 } if rank > len(sorted) { rank = len(sorted) } return sorted[rank-1] } func (provider *RealProvider) jobMetrics(ctx context.Context, window time.Duration) (governance.JobWindowMetrics, error) { windowSeconds := int(window.Seconds()) var metrics governance.JobWindowMetrics // Pull the per-outcome detail (bounded) for exact-rate and percentile math. // Duration is only measurable when the whole lifecycle lives inside the // window; older-created jobs report -1 so percentiles never mix partials. rows, err := provider.db.Query(ctx, ` SELECT state, CASE WHEN created_at > $2::timestamptz THEN COALESCE(EXTRACT(EPOCH FROM (updated_at - created_at)), -1) ELSE -1 END FROM jobs WHERE updated_at > $2::timestamptz AND updated_at <= $1::timestamptz AND state IN ('completed','failed') ORDER BY updated_at DESC LIMIT 10000`, provider.now(), provider.now().Add(-window)) if err != nil { return metrics, fmt.Errorf("query job window outcomes: %w", err) } defer rows.Close() var outcomes []windowJobOutcome for rows.Next() { var outcome windowJobOutcome if err := rows.Scan(&outcome.state, &outcome.durationSeconds); err != nil { return metrics, fmt.Errorf("scan job window outcome: %w", err) } outcomes = append(outcomes, outcome) } if err := rows.Err(); err != nil { return metrics, fmt.Errorf("iterate job window outcomes: %w", err) } metrics = computeJobWindowMetrics(outcomes, windowSeconds) // Started inside the window uses created_at (any state). var started int if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE created_at > $2::timestamptz AND created_at <= $1::timestamptz`, provider.now(), provider.now().Add(-window)).Scan(&started); err != nil { return metrics, fmt.Errorf("count jobs started: %w", err) } metrics.JobsStarted = started return metrics, nil } func (provider *RealProvider) queueGauges(ctx context.Context) (governance.QueueGauges, error) { var gauges governance.QueueGauges var oldestPendingSeconds *float64 err := provider.db.QueryRow(ctx, ` SELECT COUNT(*) FILTER (WHERE state='pending')::integer, COUNT(*) FILTER (WHERE state='running')::integer, COUNT(*) FILTER (WHERE state='retry')::integer, COUNT(*) FILTER (WHERE state='failed')::integer, CASE WHEN COUNT(*) FILTER (WHERE state IN ('pending','retry')) = 0 THEN NULL ELSE GREATEST(EXTRACT(EPOCH FROM ($1::timestamptz - MIN(created_at) FILTER (WHERE state IN ('pending','retry')))), 0) END FROM jobs`, provider.now()).Scan(&gauges.Pending, &gauges.Running, &gauges.Retry, &gauges.Failed, &oldestPendingSeconds) if err != nil { return gauges, fmt.Errorf("scan queue gauges: %w", err) } gauges.OldestPendingAgeSeconds = oldestPendingSeconds return gauges, nil } func (provider *RealProvider) activeWorkerCount(ctx context.Context) (int, error) { // Count distinct fresh leases only; never infer from container counts. var active int err := provider.db.QueryRow(ctx, ` SELECT COUNT(DISTINCT lease_owner)::integer FROM jobs WHERE state = 'running' AND lease_until > $1::timestamptz AND lease_owner IS NOT NULL`, provider.now().Add(-metricsWorkerLease)).Scan(&active) if err != nil { return 0, fmt.Errorf("count active workers: %w", err) } return active, nil } // Metrics implements the P9 run-metrics endpoint: frozen 1h/24h windows, // lease-based active workers, and null-safe samples. func (provider *RealProvider) Metrics(ctx context.Context) (governance.RunMetrics, error) { now := provider.now().UTC() metrics := governance.RunMetrics{ObservedAt: now, WorkerSource: "pg_lease"} short, err := provider.jobMetrics(ctx, metricsShortWindow) if err != nil { return metrics, err } longWindow, err := provider.jobMetrics(ctx, metricsLongWindow) if err != nil { return metrics, err } gauges, err := provider.queueGauges(ctx) if err != nil { return metrics, err } active, err := provider.activeWorkerCount(ctx) if err != nil { return metrics, err } metrics.ShortWindow = short metrics.LongWindow = longWindow metrics.Queue = gauges metrics.ActiveWorkers = active return metrics, nil }