knowledge-governance/internal/monitoring/metrics.go

package monitoring

import (
	"context"
	"fmt"
	"math"
	"sort"
	"time"

	"knowledge-governance/internal/governance"
)

type windowJobOutcome struct {
	state           string
	durationSeconds float64 // <0 when duration is unknowable
}

const (
	metricsShortWindow  = time.Hour
	metricsLongWindow   = 24 * time.Hour
	metricsWorkerLease  = 90 * time.Second // workers heartbeat every <= lease; fresher than stale leases counts active
	metricsSampleMargin = 2 * time.Second
)

func computeJobWindowMetrics(outcomes []windowJobOutcome, windowSeconds int) governance.JobWindowMetrics {
	metrics := governance.JobWindowMetrics{WindowSeconds: windowSeconds}
	var durations []float64
	for _, outcome := range outcomes {
		switch outcome.state {
		case "completed":
			metrics.JobsCompleted++
		case "failed":
			metrics.JobsFailed++
		}
		if outcome.durationSeconds >= 0 {
			durations = append(durations, outcome.durationSeconds)
		}
	}
	if finished := metrics.JobsCompleted + metrics.JobsFailed; finished > 0 {
		success := math.Round(float64(metrics.JobsCompleted)/float64(finished)*10000) / 100
		failure := math.Round(float64(metrics.JobsFailed)/float64(finished)*10000) / 100
		metrics.SuccessRate = &success
		metrics.FailureRate = &failure
	}
	if len(durations) > 0 {
		sort.Float64s(durations)
		var sum float64
		for _, value := range durations {
			sum += value
		}
		average := math.Round(sum/float64(len(durations))*10) / 10
		p50 := math.Round(percentile(durations, 0.50)*10) / 10
		p95 := math.Round(percentile(durations, 0.95)*10) / 10
		metrics.DurationAvgSeconds = &average
		metrics.DurationP50Seconds = &p50
		metrics.DurationP95Seconds = &p95
	}
	return metrics
}

// percentile uses nearest-rank so tiny samples never interpolate a fake value.
func percentile(sorted []float64, fraction float64) float64 {
	rank := int(math.Ceil(fraction * float64(len(sorted))))
	if rank < 1 {
		rank = 1
	}
	if rank > len(sorted) {
		rank = len(sorted)
	}
	return sorted[rank-1]
}

func (provider *RealProvider) jobMetrics(ctx context.Context, window time.Duration) (governance.JobWindowMetrics, error) {
	windowSeconds := int(window.Seconds())
	var metrics governance.JobWindowMetrics
	// Pull the per-outcome detail (bounded) for exact-rate and percentile math.
	// Duration is only measurable when the whole lifecycle lives inside the
	// window; older-created jobs report -1 so percentiles never mix partials.
	rows, err := provider.db.Query(ctx, `
SELECT state,
       CASE WHEN created_at > $2::timestamptz THEN COALESCE(EXTRACT(EPOCH FROM (updated_at - created_at)), -1) ELSE -1 END
FROM jobs
WHERE updated_at > $2::timestamptz AND updated_at <= $1::timestamptz
  AND state IN ('completed','failed')
ORDER BY updated_at DESC
LIMIT 10000`, provider.now(), provider.now().Add(-window))
	if err != nil {
		return metrics, fmt.Errorf("query job window outcomes: %w", err)
	}
	defer rows.Close()
	var outcomes []windowJobOutcome
	for rows.Next() {
		var outcome windowJobOutcome
		if err := rows.Scan(&outcome.state, &outcome.durationSeconds); err != nil {
			return metrics, fmt.Errorf("scan job window outcome: %w", err)
		}
		outcomes = append(outcomes, outcome)
	}
	if err := rows.Err(); err != nil {
		return metrics, fmt.Errorf("iterate job window outcomes: %w", err)
	}
	metrics = computeJobWindowMetrics(outcomes, windowSeconds)

	// Started inside the window uses created_at (any state).
	var started int
	if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE created_at > $2::timestamptz AND created_at <= $1::timestamptz`, provider.now(), provider.now().Add(-window)).Scan(&started); err != nil {
		return metrics, fmt.Errorf("count jobs started: %w", err)
	}
	metrics.JobsStarted = started
	return metrics, nil
}

func (provider *RealProvider) queueGauges(ctx context.Context) (governance.QueueGauges, error) {
	var gauges governance.QueueGauges
	var oldestPendingSeconds *float64
	err := provider.db.QueryRow(ctx, `
SELECT COUNT(*) FILTER (WHERE state='pending')::integer,
       COUNT(*) FILTER (WHERE state='running')::integer,
       COUNT(*) FILTER (WHERE state='retry')::integer,
       COUNT(*) FILTER (WHERE state='failed')::integer,
       CASE WHEN COUNT(*) FILTER (WHERE state IN ('pending','retry')) = 0 THEN NULL
            ELSE GREATEST(EXTRACT(EPOCH FROM ($1::timestamptz - MIN(created_at) FILTER (WHERE state IN ('pending','retry')))), 0) END
FROM jobs`, provider.now()).Scan(&gauges.Pending, &gauges.Running, &gauges.Retry, &gauges.Failed, &oldestPendingSeconds)
	if err != nil {
		return gauges, fmt.Errorf("scan queue gauges: %w", err)
	}
	gauges.OldestPendingAgeSeconds = oldestPendingSeconds
	return gauges, nil
}

func (provider *RealProvider) activeWorkerCount(ctx context.Context) (int, error) {
	// Count distinct fresh leases only; never infer from container counts.
	var active int
	err := provider.db.QueryRow(ctx, `
SELECT COUNT(DISTINCT lease_owner)::integer
FROM jobs
WHERE state = 'running' AND lease_until > $1::timestamptz AND lease_owner IS NOT NULL`,
		provider.now().Add(-metricsWorkerLease)).Scan(&active)
	if err != nil {
		return 0, fmt.Errorf("count active workers: %w", err)
	}
	return active, nil
}

// Metrics implements the P9 run-metrics endpoint: frozen 1h/24h windows,
// lease-based active workers, and null-safe samples.
func (provider *RealProvider) Metrics(ctx context.Context) (governance.RunMetrics, error) {
	now := provider.now().UTC()
	metrics := governance.RunMetrics{ObservedAt: now, WorkerSource: "pg_lease"}
	short, err := provider.jobMetrics(ctx, metricsShortWindow)
	if err != nil {
		return metrics, err
	}
	longWindow, err := provider.jobMetrics(ctx, metricsLongWindow)
	if err != nil {
		return metrics, err
	}
	gauges, err := provider.queueGauges(ctx)
	if err != nil {
		return metrics, err
	}
	active, err := provider.activeWorkerCount(ctx)
	if err != nil {
		return metrics, err
	}
	metrics.ShortWindow = short
	metrics.LongWindow = longWindow
	metrics.Queue = gauges
	metrics.ActiveWorkers = active
	return metrics, nil
}