Files
knowledge-governance/internal/monitoring/health.go
T
姚章浩 1db21fb899 feat(settings): 服务配置四项化并保存时真校验
- 生产/测试 WeKnora 地址、租户 ID、知识库 ID 由静态 env 收敛为版本化 UI 设置,保留修订号与操作人审计
- 新增 serviceverify:保存前真调目标端点探测,端点无效才拒绝;网络抖动或无凭据仅告警
- 迁移 000042(生产 WeKnora 端点)/000043(服务目标);补齐设置候选与校验契约用例
2026-09-30 10:54:07 +08:00

346 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package monitoring
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"sort"
"strings"
"time"
"knowledge-governance/internal/governance"
)
// P10: every configured dependency reports Liveness ("process reachable") and
// Readiness ("can accept work") independently. Missing deployment and missing
// configuration are explicit states and never collapse into "unknown".
const (
LiveUp = "up"
LiveDown = "down"
LiveUnknown = "unknown"
ReadReady = "ready"
ReadBlocked = "blocked"
ReadNotConfigured = "not_configured"
ReadNotApplicable = "not_applicable"
)
// ProbeOutcome is one raw probe attempt.
type ProbeOutcome struct {
Reachability error // liveness result (nil = reachable)
WorkReadiness error // readiness result (nil = can accept work)
Latency time.Duration
ObservedAt time.Time
}
var (
errCapacityFull = errors.New("capacity_full")
errNoPermission = errors.New("no_permission")
)
// classify turns one probe outcome into health states. Explicit not-configured
// and not-applicable results win over generic reachability failures.
func classify(key, name string, configured, applicable bool, outcome ProbeOutcome) governance.ServiceHealth {
health := governance.ServiceHealth{Key: key, Name: name, Configured: configured, Applicable: applicable, LatencyMS: outcome.Latency.Milliseconds(), ObservedAt: outcome.ObservedAt}
switch {
case !applicable:
health.Liveness, health.Reason = LiveUnknown, "未部署:本平台不使用该依赖"
health.Readiness = ReadNotApplicable
return health
case !configured:
health.Liveness, health.Reason = LiveUnknown, "已部署但未配置探测参数"
health.Readiness = ReadNotConfigured
return health
}
if outcome.Reachability != nil {
health.Readiness = ReadBlocked
health.Reason = sanitizeProbeError(outcome.Reachability)
if errors.Is(outcome.Reachability, errCapacityFull) || errors.Is(outcome.Reachability, errNoPermission) {
health.Liveness = LiveUp // reachable, but not workable
} else {
health.Liveness = LiveDown
}
return health
}
health.Liveness = LiveUp
if outcome.WorkReadiness != nil {
health.Readiness, health.Reason = ReadBlocked, sanitizeProbeError(outcome.WorkReadiness)
return health
}
health.Readiness = ReadReady
return health
}
// sanitizeProbeError keeps operators useful text without leaking endpoints,
// credentials, or hosts that live inside URLs.
func sanitizeProbeError(err error) string {
switch {
case errors.Is(err, errCapacityFull):
return "服务可达但容量已满(429),暂时不可接单"
case errors.Is(err, errNoPermission):
return "服务可达但目标知识库无读取权限"
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
return "探测超时"
}
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
return "地址解析失败(DNS)"
}
var netErr net.Error
if errors.As(err, &netErr) {
return "网络不可达"
}
message := err.Error()
if len(message) > 160 {
return message[:160]
}
return message
}
// healthTracker remembers recent success and consecutive failures per service.
type healthTracker struct {
records map[string]*trackedHealth
}
type trackedHealth struct {
lastSuccess *time.Time
failStreak int
}
func newHealthTracker() *healthTracker {
return &healthTracker{records: map[string]*trackedHealth{}}
}
func (tracker *healthTracker) apply(key string, health governance.ServiceHealth) governance.ServiceHealth {
record := tracker.records[key]
if record == nil {
record = &trackedHealth{}
tracker.records[key] = record
}
switch {
case health.Liveness == LiveUp && health.Readiness == ReadReady:
now := health.ObservedAt
record.lastSuccess = &now
record.failStreak = 0
case health.Liveness == LiveUnknown:
// Config-level states say nothing about runtime; keep the history.
default:
record.failStreak++
}
health.LastSuccessAt = record.lastSuccess
health.ConsecutiveFailures = record.failStreak
return health
}
// httpProbe performs a liveness-only HTTP GET on <base>/health, mapped through
// classify. configured=false when the base URL is absent.
func (provider *RealProvider) httpProbe(ctx context.Context, key, name, baseURL string, now time.Time) governance.ServiceHealth {
baseURL = strings.TrimSpace(baseURL)
if baseURL == "" {
return provider.track(classify(key, name, false, true, ProbeOutcome{ObservedAt: now}))
}
parsed, err := url.Parse(baseURL)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: errors.New("探测配置无效"), ObservedAt: now}))
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))
}
started := time.Now()
response, err := provider.httpClient.Do(request)
outcome := ProbeOutcome{Latency: time.Since(started), ObservedAt: now}
if err != nil {
outcome.Reachability = err
return provider.track(classify(key, name, true, true, outcome))
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode < 200 || response.StatusCode >= 300 {
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
return provider.track(classify(key, name, true, true, outcome))
}
return provider.track(classify(key, name, true, true, outcome))
}
// weKnoraProbe checks the WeKnora API: liveness via /health, readiness via an
// optional real read against a routing knowledge base.
func (provider *RealProvider) weKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth {
baseURL := provider.resolveTestWeKnoraBaseURL(ctx)
if baseURL == "" {
return provider.track(classify("weknora", "WeKnora API", false, true, ProbeOutcome{ObservedAt: now}))
}
outcome := provider.httpProbeOutcome(ctx, baseURL, now)
if outcome.Reachability == nil && provider.weknoraReadiness != nil {
outcome.WorkReadiness = provider.weknoraReadiness(ctx)
}
return provider.track(classify("weknora", "WeKnora API", true, true, outcome))
}
// productionWeKnoraProbe reports the real state of the production WeKnora
// endpoint (liveness via /health). It is advisory: publishing stays gated by the
// production switch and keeps its target in the release runtime config.
func (provider *RealProvider) productionWeKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth {
baseURL := provider.resolveProductionWeKnoraBaseURL(ctx)
if baseURL == "" {
return provider.track(classify("weknora-production", "生产 WeKnora", false, true, ProbeOutcome{ObservedAt: now}))
}
return provider.track(classify("weknora-production", "生产 WeKnora", true, true, provider.httpProbeOutcome(ctx, baseURL, now)))
}
func (provider *RealProvider) httpProbeOutcome(ctx context.Context, baseURL string, now time.Time) ProbeOutcome {
parsed, err := url.Parse(baseURL)
outcome := ProbeOutcome{ObservedAt: now}
if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" || parsed.Host == "" {
outcome.Reachability = errors.New("探测配置无效")
return outcome
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
outcome.Reachability = err
return outcome
}
started := time.Now()
response, err := provider.httpClient.Do(request)
outcome.Latency = time.Since(started)
if err != nil {
outcome.Reachability = err
return outcome
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
switch {
case response.StatusCode == http.StatusTooManyRequests:
// Reachable but at capacity: liveness stays up, readiness blocks.
outcome.Reachability = errCapacityFull
case response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized:
outcome.Reachability = errNoPermission
case response.StatusCode < 200 || response.StatusCode >= 300:
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
}
return outcome
}
// workerProbe reads queue-side worker heartbeats: no fresh heartbeat row means
// the worker loop is not touching the queue (liveness down), while the queue
// itself can still be ready for a future worker.
func (provider *RealProvider) workerProbe(ctx context.Context, now time.Time) []governance.ServiceHealth {
rows, err := provider.db.Query(ctx, `SELECT worker_id, last_seen_at FROM worker_heartbeats WHERE last_seen_at >= $1::timestamptz ORDER BY worker_id`, now.Add(-provider.workerStaleAfter))
if err != nil {
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
}
defer rows.Close()
type entry struct {
id string
seen time.Time
}
live := []entry{}
for rows.Next() {
var id string
var seen time.Time
if err := rows.Scan(&id, &seen); err != nil {
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
}
live = append(live, entry{id, seen.UTC()})
}
if len(live) == 0 {
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: errors.New("队列无新鲜心跳(worker未运行或全部停跳)"), ObservedAt: now}))}
}
out := make([]governance.ServiceHealth, 0, len(live))
for _, item := range live {
outcome := ProbeOutcome{ObservedAt: item.seen}
if now.Sub(item.seen) > provider.workerStaleAfter {
outcome.Reachability = errors.New("心跳过期")
}
out = append(out, provider.track(classify("worker:"+item.id, "解析Worker "+item.id, true, true, outcome)))
}
return out
}
func (provider *RealProvider) track(health governance.ServiceHealth) governance.ServiceHealth {
return provider.health.apply(health.Key, health)
}
// Services reports the independent Liveness/Readiness matrix for every service
// the deployment actually has (P10). Missing services are explicit states.
func (provider *RealProvider) Services(ctx context.Context) []governance.ServiceHealth {
now := provider.now().UTC()
health := make([]governance.ServiceHealth, 0, 8)
// PostgreSQL: liveness = ping, readiness = queryable.
pgOutcome := ProbeOutcome{ObservedAt: now}
started := time.Now()
if err := provider.db.Ping(ctx); err != nil {
pgOutcome.Reachability = err
} else {
pgOutcome.Latency = time.Since(started)
var ok int
if err := provider.db.QueryRow(ctx, "SELECT 1").Scan(&ok); err != nil {
pgOutcome.WorkReadiness = err
}
}
health = append(health, provider.track(classify("postgres", "PostgreSQL", true, true, pgOutcome)))
// Queue readiness rides on the jobs table.
queueOutcome := ProbeOutcome{ObservedAt: now}
var pending int
if err := provider.db.QueryRow(ctx, "SELECT COUNT(*) FROM jobs WHERE state IN ('pending','retry')").Scan(&pending); err != nil {
queueOutcome.Reachability = err
}
health = append(health, provider.track(classify("queue", "任务队列", true, true, queueOutcome)))
health = append(health, provider.workerProbe(ctx, now)...)
health = append(health, provider.weKnoraProbe(ctx, now))
health = append(health, provider.productionWeKnoraProbe(ctx, now))
health = append(health, provider.httpProbe(ctx, "mineru", "MinerU", provider.minerUBaseURL, now))
// Not-applicable dependencies stay explicit (embedding/llm managed by
// WeKnora, redis/mq/oss not deployed).
health = append(health,
provider.track(classify("embedding", "Embedding模型", false, false, ProbeOutcome{ObservedAt: now})),
provider.track(classify("llm", "LLM模型", false, false, ProbeOutcome{ObservedAt: now})),
provider.track(classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})),
provider.track(classify("oss", "OSS", false, false, ProbeOutcome{ObservedAt: now})),
)
sort.Slice(health, func(i, j int) bool { return health[i].Key < health[j].Key })
return health
}
// Readiness blocks only on services that ARE deployed and configured and
// currently fail readiness; not_configured is surfaced as a warning elsewhere.
// advisoryHealthKeys are reported to operators but never block the platform's
// own readiness: a production endpoint outage must not stop the test pipeline.
var advisoryHealthKeys = map[string]bool{"weknora-production": true}
func overallReadiness(health []governance.ServiceHealth) (bool, []string) {
blocked := []string{}
for _, item := range health {
if !item.Applicable || !item.Configured || advisoryHealthKeys[item.Key] {
continue
}
if item.Readiness != ReadReady {
blocked = append(blocked, item.Key)
}
}
return len(blocked) == 0, blocked
}
func (tracker *healthTracker) snapshot() []governance.ServiceHealth {
keys := make([]string, 0, len(tracker.records))
for key := range tracker.records {
keys = append(keys, key)
}
sort.Strings(keys)
records := make([]governance.ServiceHealth, 0, len(keys))
for _, key := range keys {
record := tracker.records[key]
item := governance.ServiceHealth{Key: key, LastSuccessAt: record.lastSuccess, ConsecutiveFailures: record.failStreak}
records = append(records, item)
}
return records
}