knowledge-governance/internal/monitoring/p9_metrics_integration_test.go

package monitoring

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"net/url"
	"os"
	"testing"
	"time"

	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgconn"
	"github.com/jackc/pgx/v5/pgxpool"

	"knowledge-governance/internal/platform/postgres"
)

type metricsPool struct{ pool *pgxpool.Pool }

func (d metricsPool) Ping(ctx context.Context) error { return d.pool.Ping(ctx) }

func (d metricsPool) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) {
	return d.pool.Query(ctx, sql, args...)
}
func (d metricsPool) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row {
	return d.pool.QueryRow(ctx, sql, args...)
}
func (d metricsPool) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) {
	return d.pool.Exec(ctx, sql, args...)
}

func isolatedMetricsURL(t *testing.T) string {
	t.Helper()
	databaseURL := os.Getenv("TEST_DATABASE_URL")
	if databaseURL == "" {
		t.Skip("TEST_DATABASE_URL is not set")
	}
	parsed, err := url.Parse(databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	digest := sha256.Sum256([]byte(t.Name()))
	schema := "kgtest_" + hex.EncodeToString(digest[:8])
	ctx := context.Background()
	admin, err := pgxpool.New(ctx, databaseURL)
	if err != nil {
		t.Fatal(err)
	}
	defer admin.Close()
	if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		cleanup, err := pgxpool.New(context.Background(), databaseURL)
		if err == nil {
			_, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
			cleanup.Close()
		}
	})
	query := parsed.Query()
	query.Set("search_path", schema)
	parsed.RawQuery = query.Encode()
	return parsed.String()
}

// TestRealMetricsWindowsAndLeases freezes docs plan P9: rates and percentiles
// come from frozen 1h/24h windows, active workers count fresh distinct leases
// only, jobs created outside the window never fake a duration, and empty
// windows return nulls instead of 0.
func TestRealMetricsWindowsAndLeases(t *testing.T) {
	ctx := context.Background()
	pool, err := pgxpool.New(ctx, isolatedMetricsURL(t))
	if err != nil {
		t.Fatal(err)
	}
	defer pool.Close()
	if err := postgres.ApplyMigrations(ctx, pool); err != nil {
		t.Fatal(err)
	}

	now := time.Date(2026, time.September, 12, 8, 0, 0, 0, time.UTC)
	seed := func(id, state string, createdAt, updatedAt time.Time, owner string, leaseUntil *time.Time, attempt int) {
		var lease any
		if leaseUntil != nil {
			lease = *leaseUntil
		}
		var ownerArg any
		if owner != "" {
			ownerArg = owner
		}
		_, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,lease_owner,lease_until,attempt,max_attempts,idempotency_key,created_at,updated_at)
VALUES($1::varchar,'test.upload','{}',$2,$4,$5,$6,$7,5,'k-'||$1::text,$3,$4)`, id, state, createdAt, updatedAt, ownerArg, lease, attempt)
		if err != nil {
			t.Fatalf("seed %s: %v", id, err)
		}
	}

	// Completed inside both windows with exact durations.
	for i, d := range []time.Duration{10 * time.Second, 20 * time.Second, 30 * time.Second, 15 * time.Minute, 30 * time.Minute} {
		updated := now.Add(-time.Duration(6-i) * time.Minute)
		seed(fmt.Sprintf("job-c%d", i), "completed", updated.Add(-d), updated, "", nil, 1)
	}
	// Failed: two inside 1h, one inside 24h only, one outside 24h.
	seed("job-f1", "failed", now.Add(-20*time.Minute), now.Add(-19*time.Minute), "w1", nil, 5)
	seed("job-f2", "failed", now.Add(-10*time.Minute), now.Add(-9*time.Minute), "w1", nil, 5)
	seed("job-f3", "failed", now.Add(-20*time.Hour), now.Add(-19*time.Hour), "w1", nil, 5)
	seed("job-f4", "failed", now.Add(-40*time.Hour), now.Add(-39*time.Hour), "w1", nil, 5)
	// Completed now but created 30h ago: duration unknowable, must not pollute percentiles.
	seed("job-old", "completed", now.Add(-30*time.Hour), now.Add(-10*time.Minute), "", nil, 1)
	// Fresh leases: two rows share wA, one wB, stale wC must not count.
	seed("job-run-a1", "running", now.Add(-5*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1)
	seed("job-run-a2", "running", now.Add(-4*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1)
	seed("job-run-b", "running", now.Add(-3*time.Minute), now.Add(-time.Minute), "wB", ptrTime(now.Add(time.Minute)), 1)
	seed("job-run-stale", "running", now.Add(-9*time.Hour), now.Add(-60*time.Minute), "wC", ptrTime(now.Add(-10*time.Minute)), 1)
	// Gauges.
	seed("job-pending", "pending", now.Add(-3*time.Hour), now.Add(-3*time.Hour), "", nil, 0)
	seed("job-retry", "retry", now.Add(-30*time.Minute), now.Add(-25*time.Minute), "", nil, 1)

	provider, err := NewRealProvider(RealProviderConfig{
		DB:  metricsPool{pool: pool},
		Now: func() time.Time { return now },
	})
	if err != nil {
		t.Fatal(err)
	}
	metrics, err := provider.Metrics(ctx)
	if err != nil {
		t.Fatalf("metrics: %v", err)
	}
	if !metrics.ObservedAt.Equal(now) || metrics.WorkerSource != "pg_lease" {
		t.Fatalf("observed/source = %s/%s", metrics.ObservedAt, metrics.WorkerSource)
	}
	if metrics.ActiveWorkers != 2 {
		t.Fatalf("active workers = %d, want 2 (fresh distinct owners only)", metrics.ActiveWorkers)
	}
	if metrics.Queue.Pending != 1 || metrics.Queue.Running != 4 || metrics.Queue.Retry != 1 || metrics.Queue.Failed != 4 {
		t.Fatalf("queue gauges = %+v", metrics.Queue)
	}
	if metrics.Queue.OldestPendingAgeSeconds == nil || int(*metrics.Queue.OldestPendingAgeSeconds) != 3*3600 {
		t.Fatalf("oldest pending = %v, want 10800s", metrics.Queue.OldestPendingAgeSeconds)
	}

	short := metrics.ShortWindow
	// completed inside 1h: 5 seeded + job-old + job-c* (all updated within 1h except none) -> 6; failed 2.
	if short.WindowSeconds != 3600 || short.JobsCompleted != 6 || short.JobsFailed != 2 {
		t.Fatalf("short counts = %+v, want 6/2", short)
	}
	if short.SuccessRate == nil || *short.SuccessRate != 75 || short.FailureRate == nil || *short.FailureRate != 25 {
		t.Fatalf("short rates = %v/%v, want 75/25", short.SuccessRate, short.FailureRate)
	}
	// Durations measurable inside 1h: 10,20,30,900,1800 (job-old excluded).
	if short.DurationAvgSeconds == nil || *short.DurationAvgSeconds != 411.4 {
		t.Fatalf("short avg = %v, want 411.4(含2条60s失败样本)", *short.DurationAvgSeconds)
	}
	if short.DurationP95Seconds == nil || *short.DurationP95Seconds != 1800 {
		t.Fatalf("short p95 = %v, want 1800", *short.DurationP95Seconds)
	}

	long := metrics.LongWindow
	if long.JobsFailed != 3 {
		t.Fatalf("long failures = %d, want 3 (job-f4 outside window)", long.JobsFailed)
	}
	// 8 measurable samples (5 completed + 2 failed@60s + job-f3 3600s): p50=60, p95=3600.
	if long.DurationP50Seconds == nil || *long.DurationP50Seconds != 60 {
		t.Fatalf("long p50 = %v, want 60", *long.DurationP50Seconds)
	}
	if long.DurationP95Seconds == nil || *long.DurationP95Seconds != 3600 {
		t.Fatalf("long p95 = %v, want 3600", *long.DurationP95Seconds)
	}

	// Empty-window honesty: a fresh schema with no rows yields null rates.
	empty, err := NewRealProvider(RealProviderConfig{DB: metricsPool{pool: pool}, Now: func() time.Time { return now.Add(-100 * time.Hour) }})
	if err != nil {
		t.Fatal(err)
	}
	idle, err := empty.Metrics(ctx)
	if err != nil {
		t.Fatalf("idle metrics: %v", err)
	}
	if idle.ShortWindow.SuccessRate != nil || idle.ShortWindow.DurationAvgSeconds != nil {
		t.Fatalf("empty window must be null, got %+v", idle.ShortWindow)
	}
	if idle.Queue.OldestPendingAgeSeconds == nil {
		t.Fatalf("oldest pending must remain observable: %+v", idle.Queue)
	}
}

func ptrTime(value time.Time) *time.Time { return &value }