1db21fb899
- 生产/测试 WeKnora 地址、租户 ID、知识库 ID 由静态 env 收敛为版本化 UI 设置,保留修订号与操作人审计 - 新增 serviceverify:保存前真调目标端点探测,端点无效才拒绝;网络抖动或无凭据仅告警 - 迁移 000042(生产 WeKnora 端点)/000043(服务目标);补齐设置候选与校验契约用例
122 lines
4.2 KiB
Go
122 lines
4.2 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"knowledge-governance/internal/platform/postgres"
|
|
)
|
|
|
|
// TestRealServicesHealthMatrix freezes docs plan P10: liveness and readiness
|
|
// are per-service and independent, worker liveness rides on queue heartbeats,
|
|
// and missing configuration is an explicit state — never "unknown".
|
|
func TestRealServicesHealthMatrix(t *testing.T) {
|
|
databaseURL := isolatedMetricsURL(t)
|
|
pool, err := pgxpool.New(context.Background(), databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer pool.Close()
|
|
ctx := context.Background()
|
|
if err := postgres.ApplyMigrations(ctx, pool); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
now := time.Date(2026, time.September, 12, 12, 0, 0, 0, time.UTC)
|
|
provider, err := NewRealProvider(RealProviderConfig{
|
|
DB: metricsPool{pool},
|
|
Now: func() time.Time { return now },
|
|
HTTPClient: &http.Client{Timeout: time.Second},
|
|
WorkerStaleAfter: 2 * time.Minute,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Before any worker claims: queue/postgres healthy, worker aggregated down.
|
|
health := provider.Services(ctx)
|
|
byKey := map[string]string{}
|
|
for _, item := range health {
|
|
byKey[item.Key+"/l"] = item.Liveness
|
|
byKey[item.Key+"/r"] = item.Readiness
|
|
}
|
|
if byKey["postgres/l"] != LiveUp || byKey["postgres/r"] != ReadReady {
|
|
t.Fatalf("postgres health = %v/%v", byKey["postgres/l"], byKey["postgres/r"])
|
|
}
|
|
if byKey["worker/l"] != LiveDown || byKey["worker/r"] != ReadBlocked {
|
|
t.Fatalf("worker health with no heartbeat = %v/%v", byKey["worker/l"], byKey["worker/r"])
|
|
}
|
|
if byKey["weknora/r"] != ReadNotConfigured || byKey["mineru/r"] != ReadNotConfigured {
|
|
t.Fatalf("unconfigured deps must be explicit: %v %v", byKey["weknora/r"], byKey["mineru/r"])
|
|
}
|
|
// Production WeKnora is probed separately; with no address configured it is
|
|
// an explicit not-configured state, never a fake "reachable".
|
|
if byKey["weknora-production/r"] != ReadNotConfigured || byKey["weknora-production/l"] != LiveUnknown {
|
|
t.Fatalf("unconfigured production endpoint = %v/%v", byKey["weknora-production/l"], byKey["weknora-production/r"])
|
|
}
|
|
for _, key := range []string{"redis", "oss", "embedding", "llm"} {
|
|
if byKey[key+"/r"] != ReadNotApplicable || byKey[key+"/l"] != LiveUnknown {
|
|
t.Fatalf("not-deployed %s = %v/%v", key, byKey[key+"/l"], byKey[key+"/r"])
|
|
}
|
|
}
|
|
// Readiness must be blocked ONLY by configured+deployed failures: worker.
|
|
ok, blocked := overallReadiness(health)
|
|
if ok || len(blocked) != 1 || blocked[0] != "worker" {
|
|
t.Fatalf("overall readiness ok=%v blocked=%v", ok, blocked)
|
|
}
|
|
|
|
// A worker claim refreshes the heartbeat row and flips liveness independently.
|
|
store := postgres.NewStore(pool)
|
|
if _, err := store.Claim(ctx, "kg-test-worker", []string{"test.upload"}, now, time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
health = provider.Services(ctx)
|
|
found := false
|
|
for _, item := range health {
|
|
if strings.HasPrefix(item.Key, "worker:") {
|
|
found = true
|
|
if item.Liveness != LiveUp || item.Readiness != ReadReady {
|
|
t.Fatalf("fresh worker health = %v/%v", item.Liveness, item.Readiness)
|
|
}
|
|
}
|
|
if item.Key == "worker" {
|
|
t.Fatalf("stale aggregate row must disappear once a worker is live")
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("live worker row missing after claim")
|
|
}
|
|
|
|
// Heartbeat older than the stale window: liveness down, and the API-side
|
|
// process itself stays independent (liveness of the API is never blocked).
|
|
if _, err := pool.Exec(ctx, `UPDATE worker_heartbeats SET last_seen_at = $1`, now.Add(-10*time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
health = provider.Services(ctx)
|
|
staleDown := false
|
|
for _, item := range health {
|
|
if (item.Key == "worker" || strings.HasPrefix(item.Key, "worker:")) && item.Liveness == LiveDown {
|
|
staleDown = true
|
|
}
|
|
}
|
|
if !staleDown {
|
|
t.Fatal("stale heartbeat must mark worker down")
|
|
}
|
|
if ok, _ := overallReadiness(health); ok {
|
|
t.Fatal("stale worker must block readiness")
|
|
}
|
|
// Tracker history: postgres was seen successful and keeps no failures.
|
|
for _, item := range health {
|
|
if item.Key == "postgres" {
|
|
if item.LastSuccessAt == nil || item.ConsecutiveFailures != 0 {
|
|
t.Fatalf("postgres tracker history = %v failures=%d", item.LastSuccessAt, item.ConsecutiveFailures)
|
|
}
|
|
}
|
|
}
|
|
}
|