package monitoring import ( "context" "errors" "fmt" "io" "net" "net/http" "net/url" "sort" "strings" "time" "knowledge-governance/internal/governance" ) // P10: every configured dependency reports Liveness ("process reachable") and // Readiness ("can accept work") independently. Missing deployment and missing // configuration are explicit states and never collapse into "unknown". const ( LiveUp = "up" LiveDown = "down" LiveUnknown = "unknown" ReadReady = "ready" ReadBlocked = "blocked" ReadNotConfigured = "not_configured" ReadNotApplicable = "not_applicable" ) // ProbeOutcome is one raw probe attempt. type ProbeOutcome struct { Reachability error // liveness result (nil = reachable) WorkReadiness error // readiness result (nil = can accept work) Latency time.Duration ObservedAt time.Time } var ( errCapacityFull = errors.New("capacity_full") errNoPermission = errors.New("no_permission") ) // classify turns one probe outcome into health states. Explicit not-configured // and not-applicable results win over generic reachability failures. func classify(key, name string, configured, applicable bool, outcome ProbeOutcome) governance.ServiceHealth { health := governance.ServiceHealth{Key: key, Name: name, Configured: configured, Applicable: applicable, LatencyMS: outcome.Latency.Milliseconds(), ObservedAt: outcome.ObservedAt} switch { case !applicable: health.Liveness, health.Reason = LiveUnknown, "未部署:本平台不使用该依赖" health.Readiness = ReadNotApplicable return health case !configured: health.Liveness, health.Reason = LiveUnknown, "已部署但未配置探测参数" health.Readiness = ReadNotConfigured return health } if outcome.Reachability != nil { health.Readiness = ReadBlocked health.Reason = sanitizeProbeError(outcome.Reachability) if errors.Is(outcome.Reachability, errCapacityFull) || errors.Is(outcome.Reachability, errNoPermission) { health.Liveness = LiveUp // reachable, but not workable } else { health.Liveness = LiveDown } return health } health.Liveness = LiveUp if outcome.WorkReadiness != nil { health.Readiness, health.Reason = ReadBlocked, sanitizeProbeError(outcome.WorkReadiness) return health } health.Readiness = ReadReady return health } // sanitizeProbeError keeps operators useful text without leaking endpoints, // credentials, or hosts that live inside URLs. func sanitizeProbeError(err error) string { switch { case errors.Is(err, errCapacityFull): return "服务可达但容量已满(429),暂时不可接单" case errors.Is(err, errNoPermission): return "服务可达但目标知识库无读取权限" case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): return "探测超时" } var dnsErr *net.DNSError if errors.As(err, &dnsErr) { return "地址解析失败(DNS)" } var netErr net.Error if errors.As(err, &netErr) { return "网络不可达" } message := err.Error() if len(message) > 160 { return message[:160] } return message } // healthTracker remembers recent success and consecutive failures per service. type healthTracker struct { records map[string]*trackedHealth } type trackedHealth struct { lastSuccess *time.Time failStreak int } func newHealthTracker() *healthTracker { return &healthTracker{records: map[string]*trackedHealth{}} } func (tracker *healthTracker) apply(key string, health governance.ServiceHealth) governance.ServiceHealth { record := tracker.records[key] if record == nil { record = &trackedHealth{} tracker.records[key] = record } switch { case health.Liveness == LiveUp && health.Readiness == ReadReady: now := health.ObservedAt record.lastSuccess = &now record.failStreak = 0 case health.Liveness == LiveUnknown: // Config-level states say nothing about runtime; keep the history. default: record.failStreak++ } health.LastSuccessAt = record.lastSuccess health.ConsecutiveFailures = record.failStreak return health } // httpProbe performs a liveness-only HTTP GET on /health, mapped through // classify. configured=false when the base URL is absent. func (provider *RealProvider) httpProbe(ctx context.Context, key, name, baseURL string, now time.Time) governance.ServiceHealth { baseURL = strings.TrimSpace(baseURL) if baseURL == "" { return provider.track(classify(key, name, false, true, ProbeOutcome{ObservedAt: now})) } parsed, err := url.Parse(baseURL) if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: errors.New("探测配置无效"), ObservedAt: now})) } parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health" parsed.RawQuery, parsed.Fragment = "", "" request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) if err != nil { return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: err, ObservedAt: now})) } started := time.Now() response, err := provider.httpClient.Do(request) outcome := ProbeOutcome{Latency: time.Since(started), ObservedAt: now} if err != nil { outcome.Reachability = err return provider.track(classify(key, name, true, true, outcome)) } defer response.Body.Close() _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) if response.StatusCode < 200 || response.StatusCode >= 300 { outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode) return provider.track(classify(key, name, true, true, outcome)) } return provider.track(classify(key, name, true, true, outcome)) } // weKnoraProbe checks the WeKnora API: liveness via /health, readiness via an // optional real read against a routing knowledge base. func (provider *RealProvider) weKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth { baseURL := provider.resolveTestWeKnoraBaseURL(ctx) if baseURL == "" { return provider.track(classify("weknora", "WeKnora API", false, true, ProbeOutcome{ObservedAt: now})) } outcome := provider.httpProbeOutcome(ctx, baseURL, now) if outcome.Reachability == nil && provider.weknoraReadiness != nil { outcome.WorkReadiness = provider.weknoraReadiness(ctx) } return provider.track(classify("weknora", "WeKnora API", true, true, outcome)) } // productionWeKnoraProbe reports the real state of the production WeKnora // endpoint (liveness via /health). It is advisory: publishing stays gated by the // production switch and keeps its target in the release runtime config. func (provider *RealProvider) productionWeKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth { baseURL := provider.resolveProductionWeKnoraBaseURL(ctx) if baseURL == "" { return provider.track(classify("weknora-production", "生产 WeKnora", false, true, ProbeOutcome{ObservedAt: now})) } return provider.track(classify("weknora-production", "生产 WeKnora", true, true, provider.httpProbeOutcome(ctx, baseURL, now))) } func (provider *RealProvider) httpProbeOutcome(ctx context.Context, baseURL string, now time.Time) ProbeOutcome { parsed, err := url.Parse(baseURL) outcome := ProbeOutcome{ObservedAt: now} if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" || parsed.Host == "" { outcome.Reachability = errors.New("探测配置无效") return outcome } parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health" parsed.RawQuery, parsed.Fragment = "", "" request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) if err != nil { outcome.Reachability = err return outcome } started := time.Now() response, err := provider.httpClient.Do(request) outcome.Latency = time.Since(started) if err != nil { outcome.Reachability = err return outcome } defer response.Body.Close() _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) switch { case response.StatusCode == http.StatusTooManyRequests: // Reachable but at capacity: liveness stays up, readiness blocks. outcome.Reachability = errCapacityFull case response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized: outcome.Reachability = errNoPermission case response.StatusCode < 200 || response.StatusCode >= 300: outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode) } return outcome } // workerProbe reads queue-side worker heartbeats: no fresh heartbeat row means // the worker loop is not touching the queue (liveness down), while the queue // itself can still be ready for a future worker. func (provider *RealProvider) workerProbe(ctx context.Context, now time.Time) []governance.ServiceHealth { rows, err := provider.db.Query(ctx, `SELECT worker_id, last_seen_at FROM worker_heartbeats WHERE last_seen_at >= $1::timestamptz ORDER BY worker_id`, now.Add(-provider.workerStaleAfter)) if err != nil { return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))} } defer rows.Close() type entry struct { id string seen time.Time } live := []entry{} for rows.Next() { var id string var seen time.Time if err := rows.Scan(&id, &seen); err != nil { return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))} } live = append(live, entry{id, seen.UTC()}) } if len(live) == 0 { return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: errors.New("队列无新鲜心跳(worker未运行或全部停跳)"), ObservedAt: now}))} } out := make([]governance.ServiceHealth, 0, len(live)) for _, item := range live { outcome := ProbeOutcome{ObservedAt: item.seen} if now.Sub(item.seen) > provider.workerStaleAfter { outcome.Reachability = errors.New("心跳过期") } out = append(out, provider.track(classify("worker:"+item.id, "解析Worker "+item.id, true, true, outcome))) } return out } func (provider *RealProvider) track(health governance.ServiceHealth) governance.ServiceHealth { return provider.health.apply(health.Key, health) } // Services reports the independent Liveness/Readiness matrix for every service // the deployment actually has (P10). Missing services are explicit states. func (provider *RealProvider) Services(ctx context.Context) []governance.ServiceHealth { now := provider.now().UTC() health := make([]governance.ServiceHealth, 0, 8) // PostgreSQL: liveness = ping, readiness = queryable. pgOutcome := ProbeOutcome{ObservedAt: now} started := time.Now() if err := provider.db.Ping(ctx); err != nil { pgOutcome.Reachability = err } else { pgOutcome.Latency = time.Since(started) var ok int if err := provider.db.QueryRow(ctx, "SELECT 1").Scan(&ok); err != nil { pgOutcome.WorkReadiness = err } } health = append(health, provider.track(classify("postgres", "PostgreSQL", true, true, pgOutcome))) // Queue readiness rides on the jobs table. queueOutcome := ProbeOutcome{ObservedAt: now} var pending int if err := provider.db.QueryRow(ctx, "SELECT COUNT(*) FROM jobs WHERE state IN ('pending','retry')").Scan(&pending); err != nil { queueOutcome.Reachability = err } health = append(health, provider.track(classify("queue", "任务队列", true, true, queueOutcome))) health = append(health, provider.workerProbe(ctx, now)...) health = append(health, provider.weKnoraProbe(ctx, now)) health = append(health, provider.productionWeKnoraProbe(ctx, now)) health = append(health, provider.httpProbe(ctx, "mineru", "MinerU", provider.minerUBaseURL, now)) // Not-applicable dependencies stay explicit (embedding/llm managed by // WeKnora, redis/mq/oss not deployed). health = append(health, provider.track(classify("embedding", "Embedding模型", false, false, ProbeOutcome{ObservedAt: now})), provider.track(classify("llm", "LLM模型", false, false, ProbeOutcome{ObservedAt: now})), provider.track(classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})), provider.track(classify("oss", "OSS", false, false, ProbeOutcome{ObservedAt: now})), ) sort.Slice(health, func(i, j int) bool { return health[i].Key < health[j].Key }) return health } // Readiness blocks only on services that ARE deployed and configured and // currently fail readiness; not_configured is surfaced as a warning elsewhere. // advisoryHealthKeys are reported to operators but never block the platform's // own readiness: a production endpoint outage must not stop the test pipeline. var advisoryHealthKeys = map[string]bool{"weknora-production": true} func overallReadiness(health []governance.ServiceHealth) (bool, []string) { blocked := []string{} for _, item := range health { if !item.Applicable || !item.Configured || advisoryHealthKeys[item.Key] { continue } if item.Readiness != ReadReady { blocked = append(blocked, item.Key) } } return len(blocked) == 0, blocked } func (tracker *healthTracker) snapshot() []governance.ServiceHealth { keys := make([]string, 0, len(tracker.records)) for key := range tracker.records { keys = append(keys, key) } sort.Strings(keys) records := make([]governance.ServiceHealth, 0, len(keys)) for _, key := range keys { record := tracker.records[key] item := governance.ServiceHealth{Key: key, LastSuccessAt: record.lastSuccess, ConsecutiveFailures: record.failStreak} records = append(records, item) } return records }