package monitoring import ( "context" "crypto/sha256" "encoding/hex" "fmt" "net/url" "os" "testing" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" "knowledge-governance/internal/platform/postgres" ) type metricsPool struct{ pool *pgxpool.Pool } func (d metricsPool) Ping(ctx context.Context) error { return d.pool.Ping(ctx) } func (d metricsPool) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { return d.pool.Query(ctx, sql, args...) } func (d metricsPool) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { return d.pool.QueryRow(ctx, sql, args...) } func (d metricsPool) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) { return d.pool.Exec(ctx, sql, args...) } func isolatedMetricsURL(t *testing.T) string { t.Helper() databaseURL := os.Getenv("TEST_DATABASE_URL") if databaseURL == "" { t.Skip("TEST_DATABASE_URL is not set") } parsed, err := url.Parse(databaseURL) if err != nil { t.Fatal(err) } digest := sha256.Sum256([]byte(t.Name())) schema := "kgtest_" + hex.EncodeToString(digest[:8]) ctx := context.Background() admin, err := pgxpool.New(ctx, databaseURL) if err != nil { t.Fatal(err) } defer admin.Close() if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { t.Fatal(err) } t.Cleanup(func() { cleanup, err := pgxpool.New(context.Background(), databaseURL) if err == nil { _, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE") cleanup.Close() } }) query := parsed.Query() query.Set("search_path", schema) parsed.RawQuery = query.Encode() return parsed.String() } // TestRealMetricsWindowsAndLeases freezes docs plan P9: rates and percentiles // come from frozen 1h/24h windows, active workers count fresh distinct leases // only, jobs created outside the window never fake a duration, and empty // windows return nulls instead of 0. func TestRealMetricsWindowsAndLeases(t *testing.T) { ctx := context.Background() pool, err := pgxpool.New(ctx, isolatedMetricsURL(t)) if err != nil { t.Fatal(err) } defer pool.Close() if err := postgres.ApplyMigrations(ctx, pool); err != nil { t.Fatal(err) } now := time.Date(2026, time.September, 12, 8, 0, 0, 0, time.UTC) seed := func(id, state string, createdAt, updatedAt time.Time, owner string, leaseUntil *time.Time, attempt int) { var lease any if leaseUntil != nil { lease = *leaseUntil } var ownerArg any if owner != "" { ownerArg = owner } _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,lease_owner,lease_until,attempt,max_attempts,idempotency_key,created_at,updated_at) VALUES($1::varchar,'test.upload','{}',$2,$4,$5,$6,$7,5,'k-'||$1::text,$3,$4)`, id, state, createdAt, updatedAt, ownerArg, lease, attempt) if err != nil { t.Fatalf("seed %s: %v", id, err) } } // Completed inside both windows with exact durations. for i, d := range []time.Duration{10 * time.Second, 20 * time.Second, 30 * time.Second, 15 * time.Minute, 30 * time.Minute} { updated := now.Add(-time.Duration(6-i) * time.Minute) seed(fmt.Sprintf("job-c%d", i), "completed", updated.Add(-d), updated, "", nil, 1) } // Failed: two inside 1h, one inside 24h only, one outside 24h. seed("job-f1", "failed", now.Add(-20*time.Minute), now.Add(-19*time.Minute), "w1", nil, 5) seed("job-f2", "failed", now.Add(-10*time.Minute), now.Add(-9*time.Minute), "w1", nil, 5) seed("job-f3", "failed", now.Add(-20*time.Hour), now.Add(-19*time.Hour), "w1", nil, 5) seed("job-f4", "failed", now.Add(-40*time.Hour), now.Add(-39*time.Hour), "w1", nil, 5) // Completed now but created 30h ago: duration unknowable, must not pollute percentiles. seed("job-old", "completed", now.Add(-30*time.Hour), now.Add(-10*time.Minute), "", nil, 1) // Fresh leases: two rows share wA, one wB, stale wC must not count. seed("job-run-a1", "running", now.Add(-5*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1) seed("job-run-a2", "running", now.Add(-4*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1) seed("job-run-b", "running", now.Add(-3*time.Minute), now.Add(-time.Minute), "wB", ptrTime(now.Add(time.Minute)), 1) seed("job-run-stale", "running", now.Add(-9*time.Hour), now.Add(-60*time.Minute), "wC", ptrTime(now.Add(-10*time.Minute)), 1) // Gauges. seed("job-pending", "pending", now.Add(-3*time.Hour), now.Add(-3*time.Hour), "", nil, 0) seed("job-retry", "retry", now.Add(-30*time.Minute), now.Add(-25*time.Minute), "", nil, 1) provider, err := NewRealProvider(RealProviderConfig{ DB: metricsPool{pool: pool}, Now: func() time.Time { return now }, }) if err != nil { t.Fatal(err) } metrics, err := provider.Metrics(ctx) if err != nil { t.Fatalf("metrics: %v", err) } if !metrics.ObservedAt.Equal(now) || metrics.WorkerSource != "pg_lease" { t.Fatalf("observed/source = %s/%s", metrics.ObservedAt, metrics.WorkerSource) } if metrics.ActiveWorkers != 2 { t.Fatalf("active workers = %d, want 2 (fresh distinct owners only)", metrics.ActiveWorkers) } if metrics.Queue.Pending != 1 || metrics.Queue.Running != 4 || metrics.Queue.Retry != 1 || metrics.Queue.Failed != 4 { t.Fatalf("queue gauges = %+v", metrics.Queue) } if metrics.Queue.OldestPendingAgeSeconds == nil || int(*metrics.Queue.OldestPendingAgeSeconds) != 3*3600 { t.Fatalf("oldest pending = %v, want 10800s", metrics.Queue.OldestPendingAgeSeconds) } short := metrics.ShortWindow // completed inside 1h: 5 seeded + job-old + job-c* (all updated within 1h except none) -> 6; failed 2. if short.WindowSeconds != 3600 || short.JobsCompleted != 6 || short.JobsFailed != 2 { t.Fatalf("short counts = %+v, want 6/2", short) } if short.SuccessRate == nil || *short.SuccessRate != 75 || short.FailureRate == nil || *short.FailureRate != 25 { t.Fatalf("short rates = %v/%v, want 75/25", short.SuccessRate, short.FailureRate) } // Durations measurable inside 1h: 10,20,30,900,1800 (job-old excluded). if short.DurationAvgSeconds == nil || *short.DurationAvgSeconds != 411.4 { t.Fatalf("short avg = %v, want 411.4(含2条60s失败样本)", *short.DurationAvgSeconds) } if short.DurationP95Seconds == nil || *short.DurationP95Seconds != 1800 { t.Fatalf("short p95 = %v, want 1800", *short.DurationP95Seconds) } long := metrics.LongWindow if long.JobsFailed != 3 { t.Fatalf("long failures = %d, want 3 (job-f4 outside window)", long.JobsFailed) } // 8 measurable samples (5 completed + 2 failed@60s + job-f3 3600s): p50=60, p95=3600. if long.DurationP50Seconds == nil || *long.DurationP50Seconds != 60 { t.Fatalf("long p50 = %v, want 60", *long.DurationP50Seconds) } if long.DurationP95Seconds == nil || *long.DurationP95Seconds != 3600 { t.Fatalf("long p95 = %v, want 3600", *long.DurationP95Seconds) } // Empty-window honesty: a fresh schema with no rows yields null rates. empty, err := NewRealProvider(RealProviderConfig{DB: metricsPool{pool: pool}, Now: func() time.Time { return now.Add(-100 * time.Hour) }}) if err != nil { t.Fatal(err) } idle, err := empty.Metrics(ctx) if err != nil { t.Fatalf("idle metrics: %v", err) } if idle.ShortWindow.SuccessRate != nil || idle.ShortWindow.DurationAvgSeconds != nil { t.Fatalf("empty window must be null, got %+v", idle.ShortWindow) } if idle.Queue.OldestPendingAgeSeconds == nil { t.Fatalf("oldest pending must remain observable: %+v", idle.Queue) } } func ptrTime(value time.Time) *time.Time { return &value }