package monitoring import ( "context" "net/http" "strings" "testing" "time" "github.com/jackc/pgx/v5/pgxpool" "knowledge-governance/internal/platform/postgres" ) // TestRealServicesHealthMatrix freezes docs plan P10: liveness and readiness // are per-service and independent, worker liveness rides on queue heartbeats, // and missing configuration is an explicit state — never "unknown". func TestRealServicesHealthMatrix(t *testing.T) { databaseURL := isolatedMetricsURL(t) pool, err := pgxpool.New(context.Background(), databaseURL) if err != nil { t.Fatal(err) } defer pool.Close() ctx := context.Background() if err := postgres.ApplyMigrations(ctx, pool); err != nil { t.Fatal(err) } now := time.Date(2026, time.September, 12, 12, 0, 0, 0, time.UTC) provider, err := NewRealProvider(RealProviderConfig{ DB: metricsPool{pool}, Now: func() time.Time { return now }, HTTPClient: &http.Client{Timeout: time.Second}, WorkerStaleAfter: 2 * time.Minute, }) if err != nil { t.Fatal(err) } // Before any worker claims: queue/postgres healthy, worker aggregated down. health := provider.Services(ctx) byKey := map[string]string{} for _, item := range health { byKey[item.Key+"/l"] = item.Liveness byKey[item.Key+"/r"] = item.Readiness } if byKey["postgres/l"] != LiveUp || byKey["postgres/r"] != ReadReady { t.Fatalf("postgres health = %v/%v", byKey["postgres/l"], byKey["postgres/r"]) } if byKey["worker/l"] != LiveDown || byKey["worker/r"] != ReadBlocked { t.Fatalf("worker health with no heartbeat = %v/%v", byKey["worker/l"], byKey["worker/r"]) } if byKey["weknora/r"] != ReadNotConfigured || byKey["mineru/r"] != ReadNotConfigured { t.Fatalf("unconfigured deps must be explicit: %v %v", byKey["weknora/r"], byKey["mineru/r"]) } // Production WeKnora is probed separately; with no address configured it is // an explicit not-configured state, never a fake "reachable". if byKey["weknora-production/r"] != ReadNotConfigured || byKey["weknora-production/l"] != LiveUnknown { t.Fatalf("unconfigured production endpoint = %v/%v", byKey["weknora-production/l"], byKey["weknora-production/r"]) } for _, key := range []string{"redis", "oss", "embedding", "llm"} { if byKey[key+"/r"] != ReadNotApplicable || byKey[key+"/l"] != LiveUnknown { t.Fatalf("not-deployed %s = %v/%v", key, byKey[key+"/l"], byKey[key+"/r"]) } } // Readiness must be blocked ONLY by configured+deployed failures: worker. ok, blocked := overallReadiness(health) if ok || len(blocked) != 1 || blocked[0] != "worker" { t.Fatalf("overall readiness ok=%v blocked=%v", ok, blocked) } // A worker claim refreshes the heartbeat row and flips liveness independently. store := postgres.NewStore(pool) if _, err := store.Claim(ctx, "kg-test-worker", []string{"test.upload"}, now, time.Minute); err != nil { t.Fatal(err) } health = provider.Services(ctx) found := false for _, item := range health { if strings.HasPrefix(item.Key, "worker:") { found = true if item.Liveness != LiveUp || item.Readiness != ReadReady { t.Fatalf("fresh worker health = %v/%v", item.Liveness, item.Readiness) } } if item.Key == "worker" { t.Fatalf("stale aggregate row must disappear once a worker is live") } } if !found { t.Fatal("live worker row missing after claim") } // Heartbeat older than the stale window: liveness down, and the API-side // process itself stays independent (liveness of the API is never blocked). if _, err := pool.Exec(ctx, `UPDATE worker_heartbeats SET last_seen_at = $1`, now.Add(-10*time.Minute)); err != nil { t.Fatal(err) } health = provider.Services(ctx) staleDown := false for _, item := range health { if (item.Key == "worker" || strings.HasPrefix(item.Key, "worker:")) && item.Liveness == LiveDown { staleDown = true } } if !staleDown { t.Fatal("stale heartbeat must mark worker down") } if ok, _ := overallReadiness(health); ok { t.Fatal("stale worker must block readiness") } // Tracker history: postgres was seen successful and keeps no failures. for _, item := range health { if item.Key == "postgres" { if item.LastSuccessAt == nil || item.ConsecutiveFailures != 0 { t.Fatalf("postgres tracker history = %v failures=%d", item.LastSuccessAt, item.ConsecutiveFailures) } } } }